00000100:00080000:0.0F:1775773327.083206:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) 89584a75-b38e-4303-b812-ccf0f9d4c217->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775773327.083221:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1775773327.083223:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775773327.083225:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c6488034000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775773327.083229:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775773327.083234:0:96535:0:(import.c:509:import_select_connection()) MGC192.168.201.105@tcp: connect to NID 192.168.201.105@tcp last attempt 9603 00000400:00000200:0.0:1775773327.083242:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773327.083247:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:0.0:1775773327.083250:0:96535:0:(import.c:588:import_select_connection()) MGC192.168.201.105@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775773327.083253:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083255:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.083257:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083259:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:0.0:1775773327.083269:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775773327.083270:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56c88 00000020:00000001:0.0:1775773327.083272:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775773327.083278:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c6488034800 refcount=3 00000020:00000001:0.0:1775773327.083281:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554325420032 : -109519384131584 : ffff9c6488034800) 00000020:00000001:0.0:1775773327.083283:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554325420032 : -109519384131584 : ffff9c6488034800) 00000100:00000040:0.0:1775773327.083287:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083289:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.083291:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083293:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:0.0:1775773327.083297:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c6488034800 : new refcount 2 00000100:00080000:0.0:1775773327.083300:0:96535:0:(import.c:623:import_select_connection()) MGC192.168.201.105@tcp: import ffff9c6488034000 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083303:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773327.083307:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775773327.083311:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775773327.083314:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083316:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:0.0:1775773327.083319:0:96535:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775773327.083325:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c64a5fd0e00. 00000020:00000040:0.0:1775773327.083336:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c6488034000 refcount=4 obd=MGC192.168.201.105@tcp 02000000:00000001:0.0:1775773327.083340:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775773327.083342:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083345:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775773327.083349:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775773327.083353:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1775773327.083371:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c648b99a000. 00000100:00000001:0.0:1775773327.083378:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775773327.083388:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775773327.083400:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775773327.083405:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775773327.083410:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bfa6c076] to pc [ptlrpcd_rcv+-1] req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773327.083439:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775773327.083443:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) d4b1039f-ed2d-40d9-b26a-50f3c64e108a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775773327.083451:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1775773327.083453:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775773327.083455:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c64890b5800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1775773327.083458:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1775773327.083461:0:96535:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9c648a583800: connect to NID 192.168.201.105@tcp last attempt 9603 00000400:00000200:0.0:1775773327.083466:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773327.083469:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:0.0:1775773327.083472:0:96535:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9c648a583800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1775773327.083474:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083476:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.083478:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083479:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:0.0:1775773327.083481:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775773327.083482:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56cd5 00000020:00000001:0.0:1775773327.083484:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0F:1775773327.083488:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1775773327.083492:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c6484f70000 refcount=5 00000020:00000001:0.0:1775773327.083494:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554274283520 : -109519435268096 : ffff9c6484f70000) 00000020:00000001:0.0:1775773327.083495:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554274283520 : -109519435268096 : ffff9c6484f70000) 00000100:00000001:2.0:1775773327.083502:0:2357:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775773327.083506:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775773327.083506:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083511:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.083516:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000040:2.0:1775773327.083518:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773327.083519:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:0.0:1775773327.083526:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c6484f70000 : new refcount 4 00000100:00080000:0.0:1775773327.083533:0:96535:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9c648a583800: import ffff9c64890b5800 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:2.0:1775773327.083534:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775773327.083537:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083540:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773327.083543:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773327.083544:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775773327.083547:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773327.083550:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773327.083552:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1775773327.083555:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c64a5fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:89584a75-b38e-4303-b812-ccf0f9d4c217:2357:1862023308942464:192.168.201.105@tcp:250:kworker.0 02000000:00000001:0.0:1775773327.083556:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083563:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1775773327.083563:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 02000000:00000001:2.0:1775773327.083570:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1775773327.083571:0:96535:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1775773327.083573:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1775773327.083576:0:96535:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773327.083577:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1775773327.083581:0:96535:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773327.083582:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6493d11000. 02000000:00000001:2.0:1775773327.083585:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1775773327.083586:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c64a5fd0a80. 00000020:00000040:0.0:1775773327.083590:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c64890b5800 refcount=6 obd=lustre-MDT0000-mdc-ffff9c648a583800 00000400:00000200:2.0:1775773327.083593:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 02000000:00000001:0.0:1775773327.083597:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000400:00000010:2.0:1775773327.083599:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c6484b761b8. 02000000:00000001:0.0:1775773327.083600:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083605:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775773327.083606:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1775773327.083608:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f64440. 02000000:00000001:0.0:1775773327.083609:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775773327.083615:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308942464, portal 25 02000000:00000010:0.0:1775773327.083616:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c648b99bc00. 00000100:00000001:2.0:1775773327.083622:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775773327.083623:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083625:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554828328448 : -109518881223168 : ffff9c64a5fd0e00) 00000100:00100000:0.0:1775773327.083645:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1775773327.083651:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773327.083662:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775773327.083667:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083672:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775773327.083676:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f0176542] to pc [ptlrpcd_rcv+-1] req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1775773327.083680:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:2.0:1775773327.083684:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f64aa0. 00000100:00000200:2.0:1775773327.083692:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862023308942464, offset 0 00000400:00000200:2.0:1775773327.083700:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:2.0:1775773327.083713:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773327.083714:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775773327.083718:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) d4b1039f-ed2d-40d9-b26a-50f3c64e108a->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1775773327.083723:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000100:00080000:0.0:1775773327.083724:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1775773327.083729:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1775773327.083731:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c64890b0000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1775773327.083732:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1775773327.083737:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000400:00000200:2.0:1775773327.083741:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000100:00080000:0.0:1775773327.083743:0:96535:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9c648a583800: connect to NID 192.168.201.105@tcp last attempt 9603 00000400:00000200:0.0:1775773327.083751:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773327.083755:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775773327.083758:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:0.0:1775773327.083762:0:96535:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9c648a583800: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1775773327.083764:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000100:00000040:0.0:1775773327.083771:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000400:00000200:2.0:1775773327.083777:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337735:337735:256:4294967295] 192.168.201.105@tcp LPNI seq info [337680:337680:8:4294967295] 00000100:00000001:0.0:1775773327.083777:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.083784:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000400:00000200:2.0:1775773327.083790:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773327.083790:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:0.0:1775773327.083796:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775773327.083798:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56ce3 00000020:00000001:0.0:1775773327.083801:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775773327.083804:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c649f46a000 refcount=4 00000020:00000001:0.0:1775773327.083809:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554715709440 : -109518993842176 : ffff9c649f46a000) 00000400:00000200:2.0:1775773327.083814:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000020:00000001:0.0:1775773327.083815:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554715709440 : -109518993842176 : ffff9c649f46a000) 00000800:00000200:2.0:1775773327.083826:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:2.0:1775773327.083833:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c64ab0e0800. 00000100:00000040:0.0:1775773327.083844:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000800:00000200:2.0:1775773327.083845:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000100:00000001:0.0:1775773327.083851:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.1F:1775773327.083855:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000100:00000040:0.0:1775773327.083860:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083865:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:0.0:1775773327.083869:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c649f46a000 : new refcount 3 00000100:00080000:0.0:1775773327.083878:0:96535:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9c648a583800: import ffff9c64890b0000 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:0.0:1775773327.083881:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773327.083891:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1775773327.083895:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775773327.083896:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773327.083899:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083903:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000400:00000200:2.0:1775773327.083906:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000008:00100000:0.0:1775773327.083908:0:96535:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1775773327.083911:0:2357:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS new deadline estimation for cpu 0 is 4304276872 00000008:00000001:0.0:1775773327.083912:0:96535:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083918:0:96535:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773327.083921:0:2357:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c64b2e37cc0. 00000100:00000010:0.0:1775773327.083923:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c64a5fd3480. 00000020:00000040:0.0:1775773327.083928:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c64890b0000 refcount=3 obd=lustre-OST0000-osc-ffff9c648a583800 02000000:00000001:0.0:1775773327.083931:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1775773327.083932:0:2357:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1775773327.083933:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.083938:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1775773327.083939:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1775773327.083942:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083943:0:2357:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS new deadline estimation for cpu 1 is 4304276872 02000000:00000010:0.0:1775773327.083948:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c648b99b400. 00000100:00000010:2.0:1775773327.083950:0:2357:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c64b2e37de0. 00000100:00000001:0.0:1775773327.083954:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083955:0:2357:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1775773327.083962:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1775773327.083965:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773327.083967:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083974:0:2357:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083979:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773327.083980:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1775773327.083985:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.083989:0:2357:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775773327.083992:0:2357:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775773327.083995:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775773327.083997:0:2357:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1775773327.083997:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000040426db5] to pc [ptlrpcd_rcv+-1] req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773327.084000:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775773327.084008:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773327.084013:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775773327.084019:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) d4b1039f-ed2d-40d9-b26a-50f3c64e108a->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1775773327.084025:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1775773327.084027:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775773327.084028:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.084030:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1775773327.084033:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1775773327.084033:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c649f468000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1775773327.084036:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775773327.084038:0:96535:0:(import.c:484:import_select_connection()) Process entered 02000000:00000001:2.0:1775773327.084041:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775773327.084044:0:96535:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9c648a583800: connect to NID 192.168.201.105@tcp last attempt 9603 00000400:00000200:0.0:1775773327.084051:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00100000:2.0:1775773327.084056:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c64a5fd0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308942592:192.168.201.105@tcp:38:kworker.0 00000100:00000001:2.0:1775773327.084061:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775773327.084063:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:0.0:1775773327.084063:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 02000000:00000001:2.0:1775773327.084067:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1775773327.084069:0:96535:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9c648a583800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1775773327.084073:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775773327.084078:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6493d13000. 02000000:00000001:2.0:1775773327.084082:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775773327.084084:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000400:00000200:2.0:1775773327.084089:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773327.084089:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.084095:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000400:00000010:2.0:1775773327.084096:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c6484b76840. 00000100:00000001:0.0:1775773327.084099:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000400:00000010:2.0:1775773327.084103:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f64198. 00000020:00000001:0.0:1775773327.084104:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1775773327.084107:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56cea 00000100:00000200:2.0:1775773327.084110:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308942592, portal 10 00000020:00000001:0.0:1775773327.084110:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1775773327.084114:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c649f46f000 refcount=4 00000100:00000001:2.0:1775773327.084115:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1775773327.084118:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554715729920 : -109518993821696 : ffff9c649f46f000) 00000100:00000001:2.0:1775773327.084119:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554828327552 : -109518881224064 : ffff9c64a5fd0a80) 00000020:00000001:0.0:1775773327.084124:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554715729920 : -109518993821696 : ffff9c649f46f000) 00000100:00000040:0.0:1775773327.084129:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000040:2.0:1775773327.084132:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773327.084133:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1775773327.084139:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:0.0:1775773327.084145:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:0.0:1775773327.084150:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c649f46f000 : new refcount 3 00000100:00000001:2.0:1775773327.084152:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1775773327.084158:0:96535:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9c648a583800: import ffff9c649f468000 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000040:2.0:1775773327.084159:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000100:00000001:0.0:1775773327.084188:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773327.084191:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1775773327.084195:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f646e8. 02000000:00000001:0.0:1775773327.084195:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775773327.084201:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1775773327.084202:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862023308942592, offset 0 00000100:00000001:0.0:1775773327.084203:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:0.0:1775773327.084208:0:96535:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1775773327.084213:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000008:00000001:0.0:1775773327.084213:0:96535:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773327.084218:0:96535:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775773327.084222:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000010:0.0:1775773327.084225:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c64a5fd1f80. 00000400:00000200:2.0:1775773327.084229:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000020:00000040:0.0:1775773327.084231:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c649f468000 refcount=3 obd=lustre-OST0001-osc-ffff9c648a583800 00000400:00000200:2.0:1775773327.084235:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1775773327.084237:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1775773327.084240:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775773327.084241:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000100:00000001:0.0:1775773327.084245:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1775773327.084247:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1775773327.084249:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1775773327.084252:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 02000000:00000001:0.0:1775773327.084254:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775773327.084260:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337736:337736:255:4294967295] 192.168.201.105@tcp LPNI seq info [337681:337681:7:4294967295] 02000000:00000010:0.0:1775773327.084263:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c648b998000. 00000400:00000200:2.0:1775773327.084266:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773327.084275:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000100:00000001:0.0:1775773327.084281:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1775773327.084282:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:2.0:1775773327.084290:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c64ab0e0600. 00000100:00100000:0.0:1775773327.084292:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1775773327.084304:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000800:00000200:2.0:1775773327.084309:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000100:00000001:0.0:1775773327.084310:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1775773327.084318:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000100:00000040:0.0:1775773327.084319:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008b7336a2] to pc [ptlrpcd_rcv+-1] req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773327.084324:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773327.084331:0:2357:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: using a timeout of 33957 usecs (34 jiffies) 00000100:00000001:0.0:1775773327.084335:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775773327.084340:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:2.0:1775773327.084344:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS new deadline estimation for cpu 0 is 4304276003 00000100:00000040:0.0:1775773327.084344:0:96535:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (9613) 00000100:00000001:2.0:1775773327.084349:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS request active for cpu 0, keep current deadline 4304276872 00000100:00000001:2.0:1775773327.084353:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS new deadline estimation for cpu 1 is 4304276003 00000100:00000001:2.0:1775773327.084356:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS request active for cpu 1, keep current deadline 4304276872 00000100:00000001:2.0:1775773327.084359:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775773327.084362:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.084366:0:2357:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.084368:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775773327.084373:0:2357:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775773327.084375:0:2357:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775773327.084377:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775773327.084379:0:2357:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775773327.084380:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775773327.084385:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773327.084394:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775773327.084395:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773327.084398:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775773327.084400:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773327.084402:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773327.084405:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c64a5fd3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308942720:192.168.201.105@tcp:8:kworker.0 00000100:00000001:2.0:1775773327.084408:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775773327.084410:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775773327.084412:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773327.084413:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775773327.084417:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6493d12800. 02000000:00000001:2.0:1775773327.084419:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775773327.084422:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000010:2.0:1775773327.084432:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c6484b76058. 00000400:00000010:2.0:1775773327.084436:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f644c8. 00000100:00000200:2.0:1775773327.084440:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308942720, portal 4 00000100:00000001:2.0:1775773327.084442:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775773327.084443:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554828338304 : -109518881213312 : ffff9c64a5fd3480) 00000100:00000040:2.0:1775773327.084451:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773327.084457:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775773327.084460:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:2.0:1775773327.084463:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f64ee0. 00000100:00000200:2.0:1775773327.084466:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862023308942720, offset 0 00000400:00000200:2.0:1775773327.084472:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:2.0:1775773327.084476:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773327.084480:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000400:00000200:2.0:1775773327.084483:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775773327.084486:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000400:00000200:2.0:1775773327.084489:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775773327.084492:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000400:00000200:2.0:1775773327.084495:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337737:337737:254:4294967295] 192.168.201.105@tcp LPNI seq info [337682:337682:6:4294967295] 00000400:00000200:2.0:1775773327.084500:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773327.084506:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000800:00000200:2.0:1775773327.084512:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:2.0:1775773327.084516:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c64ab0e0500. 00000800:00000200:2.0:1775773327.084521:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:2.1:1775773327.084524:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000100:00000001:2.0:1775773327.084527:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773327.084530:0:2357:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: using a timeout of 161587 usecs (162 jiffies) 00000400:00000200:2.0:1775773327.084534:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:2.0:1775773327.084536:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 0 is 4304276131 00000100:00000001:2.0:1775773327.084538:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS request active for cpu 0, keep current deadline 4304276872 00000100:00000001:2.0:1775773327.084540:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 1 is 4304276131 00000100:00000001:2.0:1775773327.084549:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS request active for cpu 1, keep current deadline 4304276872 00000100:00000001:2.0:1775773327.084551:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775773327.084554:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.084556:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1775773327.084561:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773327.084569:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1775773327.084570:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773327.084572:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1775773327.084574:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773327.084576:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773327.084579:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c64a5fd1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308942848:192.168.201.105@tcp:8:kworker.0 00000100:00000001:2.0:1775773327.084582:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1775773327.084584:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1775773327.084586:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773327.084588:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1775773327.084591:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6493d12000. 02000000:00000001:2.0:1775773327.084594:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1775773327.084598:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000010:2.0:1775773327.084601:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c6484b76580. 00000400:00000010:2.0:1775773327.084605:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f642a8. 00000100:00000200:2.0:1775773327.084609:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308942848, portal 4 00000100:00000001:2.0:1775773327.084611:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1775773327.084612:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554828332928 : -109518881218688 : ffff9c64a5fd1f80) 00000100:00000040:2.0:1775773327.084619:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773327.084626:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1775773327.084645:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:2.0:1775773327.084648:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c64b2f64cc0. 00000100:00000200:2.0:1775773327.084651:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862023308942848, offset 0 00000400:00000200:2.0:1775773327.084657:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:2.0:1775773327.084661:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773327.084671:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000400:00000200:2.0:1775773327.084674:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1775773327.084677:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000400:00000200:2.0:1775773327.084680:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1775773327.084683:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000400:00000200:2.0:1775773327.084686:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337738:337738:253:4294967295] 192.168.201.105@tcp LPNI seq info [337683:337683:5:4294967295] 00000400:00000200:2.0:1775773327.084690:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773327.084698:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000800:00000200:2.0:1775773327.084703:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:2.0:1775773327.084707:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c64ab0e0a00. 00000800:00000200:2.0:1775773327.084711:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:2.1:1775773327.084715:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000100:00000001:2.0:1775773327.084719:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773327.084722:0:2357:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: using a timeout of 170937 usecs (171 jiffies) 00000400:00000200:2.0:1775773327.084726:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:2.0:1775773327.084728:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 0 is 4304276140 00000100:00000001:2.0:1775773327.084731:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS request active for cpu 0, keep current deadline 4304276872 00000100:00000001:2.0:1775773327.084734:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 1 is 4304276140 00000100:00000001:2.0:1775773327.084736:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS request active for cpu 1, keep current deadline 4304276872 00000100:00000001:2.0:1775773327.084738:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1775773327.084741:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.084744:0:2357:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.084747:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1775773327.084751:0:2357:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1775773327.084753:0:2357:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1775773327.084755:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1775773327.084757:0:2357:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1775773327.084759:0:2357:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773327.084761:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0F:1775773327.085099:0:2348:0:(lib-socket.c:454:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.105:988 00000400:00000200:3.0:1775773327.085198:0:2348:0:(lib-socket.c:454:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.105:988 00010000:00000001:0.0:1775773327.722246:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773327.722251:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773327.722256:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773327.722257:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775773327.722313:0:190496:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775773327.722317:0:190496:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775773327.722321:0:190496:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0F:1775773328.042216:0:174851:0:(connection.c:75:cpu_latency_work()) work item of ffff9c648a301910 (cpu 0) has reached its deadline 4304276872, at 4304276928 00000100:00000010:1.0:1775773328.042233:0:174851:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c64b2e37de0. 00000100:00000040:0.0:1775773328.042255:0:160383:0:(connection.c:75:cpu_latency_work()) work item of ffff9c648a301800 (cpu 0) has reached its deadline 4304276872, at 4304276928 00000100:00000010:0.0:1775773328.042262:0:160383:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c64b2e37cc0. 00010000:00000001:0.0:1775773328.746329:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773328.746336:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773328.746341:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773328.746342:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775773328.746427:0:181746:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775773328.746431:0:181746:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775773328.746434:0:181746:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775773329.642617:0:2348:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.201.105:988 00000400:00000100:2.0:1775773329.642663:0:2348:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.201.105@tcp at host 192.168.201.105 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1775773329.642673:0:2348:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773329.642692:0:2348:0:(router.c:1750:lnet_notify()) 192.168.201.5@tcp notifying 192.168.201.105@tcp: down 00000400:00000200:2.0:1775773329.642697:0:2348:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773329.642701:0:2348:0:(router.c:1835:lnet_notify()) Old news 00000800:00000100:2.0:1775773329.642708:0:2348:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.105@tcp 00000800:00000010:2.0:1775773329.642712:0:2348:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c64ab0e0800. 00000400:00000200:2.0:1775773329.642717:0:2348:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775773329.642723:0:2348:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775773329.642730:0:2348:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f64aa0 00000400:00000010:2.0:1775773329.642733:0:2348:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f64aa0. 00000100:00000001:2.0:1775773329.642739:0:2348:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775773329.642748:0:2348:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642780:0:2348:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775773329.642785:0:2348:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642794:0:2348:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773329.642796:0:2348:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775773329.642806:0:2348:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.105@tcp 00000800:00000010:2.0:1775773329.642809:0:2348:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c64ab0e0600. 00000400:00000200:2.0:1775773329.642812:0:2348:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775773329.642816:0:2348:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775773329.642819:0:2348:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f646e8 00000400:00000010:2.0:1775773329.642821:0:2348:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f646e8. 00000100:00000001:2.0:1775773329.642823:0:2348:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775773329.642836:0:2348:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642846:0:2348:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775773329.642850:0:2348:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642857:0:2348:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773329.642859:0:2348:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775773329.642865:0:2348:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.105@tcp 00000800:00000010:2.0:1775773329.642874:0:2348:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c64ab0e0500. 00000400:00000200:2.0:1775773329.642877:0:2348:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775773329.642881:0:2348:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775773329.642884:0:2348:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f64ee0 00000400:00000010:2.0:1775773329.642886:0:2348:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f64ee0. 00000100:00000001:2.0:1775773329.642889:0:2348:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775773329.642894:0:2348:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642903:0:2348:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775773329.642907:0:2348:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642914:0:2348:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773329.642916:0:2348:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1775773329.642921:0:2348:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.105@tcp 00000800:00000010:2.0:1775773329.642923:0:2348:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c64ab0e0a00. 00000400:00000200:2.0:1775773329.642926:0:2348:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1775773329.642930:0:2348:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1775773329.642933:0:2348:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f64cc0 00000400:00000010:2.0:1775773329.642935:0:2348:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f64cc0. 00000100:00000001:2.0:1775773329.642937:0:2348:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1775773329.642942:0:2348:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642950:0:2348:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1775773329.642953:0:2348:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773352 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773329.642960:0:2348:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773329.642962:0:2348:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775773329.642989:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775773329.642995:0:2357:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775773329.642998:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773329.643008:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773327/real 1775773329] req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773329.643023:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773329.643025:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773329.643030:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c6484b76058. 00000400:00000200:0.0:1775773329.643034:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f644c8 00000400:00000010:0.0:1775773329.643036:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f644c8. 00000100:00000001:0.0:1775773329.643040:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773329.643044:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773329.643070:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643078:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773329.643081:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643084:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643087:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773329.643092:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643100:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773329.643106:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643114:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773329.643118:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643120:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643121:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643124:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773329.643127:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773329.643131:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c64890b0000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773329.643138:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773329.643141:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773329.643145:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773329.643154:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c64a5fd3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308942720:192.168.201.105@tcp:8:kworker.0 00000100:00000001:0.0:1775773329.643160:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773329.643195:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773327/real 1775773329] req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773329.643217:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773329.643219:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773329.643222:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c6484b76580. 00000400:00000200:0.0:1775773329.643224:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f642a8 00000400:00000010:0.0:1775773329.643226:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f642a8. 00000100:00000001:0.0:1775773329.643229:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773329.643233:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773329.643240:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643246:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773329.643248:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643250:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643251:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773329.643255:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643261:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773329.643264:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643271:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773329.643273:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643275:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643276:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643278:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773329.643280:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773329.643283:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c649f468000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773329.643287:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773329.643290:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773329.643301:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773329.643309:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c64a5fd1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308942848:192.168.201.105@tcp:8:kworker.0 00000100:00000001:0.0:1775773329.643314:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773329.643318:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773327/real 1775773329] req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773329.643327:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773329.643329:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773329.643331:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c6484b76840. 00000400:00000200:0.0:1775773329.643334:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f64198 00000400:00000010:0.0:1775773329.643336:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f64198. 00000100:00000001:0.0:1775773329.643339:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773329.643343:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773329.643353:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643362:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773329.643364:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643367:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643368:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773329.643373:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643379:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773329.643383:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643390:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773329.643393:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643402:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643404:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643406:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773329.643408:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773329.643410:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c64890b5800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773329.643416:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773329.643418:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773329.643422:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773329.643430:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c64a5fd0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308942592:192.168.201.105@tcp:38:kworker.0 00000100:00000001:0.0:1775773329.643434:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773329.643438:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773327/real 1775773329] req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773329.643447:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773329.643448:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773329.643450:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c6484b761b8. 00000400:00000200:0.0:1775773329.643453:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c64b2f64440 00000400:00000010:0.0:1775773329.643455:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c64b2f64440. 00000100:00000001:0.0:1775773329.643457:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773329.643461:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773329.643470:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643477:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773329.643479:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643482:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643483:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773329.643488:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643509:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773329.643514:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643522:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773329.643525:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643528:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773329.643529:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643531:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773329.643534:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773329.643537:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c6488034000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773329.643542:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773329.643545:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773329.643550:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773329.643559:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c64a5fd0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:89584a75-b38e-4303-b812-ccf0f9d4c217:2357:1862023308942464:192.168.201.105@tcp:250:kworker.0 00000100:00000001:0.0:1775773329.643563:0:2357:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643566:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773329.643569:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c64a5fd3480 x1862023308942720/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643578:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773329.643601:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773329.643608:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6493d12800. 02000000:00000001:0.0:1775773329.643612:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773329.643616:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773329.643618:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c64890b0000 refcount=2 obd=lustre-OST0000-osc-ffff9c648a583800 00000020:00000001:0.0:1775773329.643620:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773329.643624:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c648b99b400. 02000000:00000001:0.0:1775773329.643629:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773329.643638:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773329.643642:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c64a5fd3480. 00000100:00000001:0.0:1775773329.643646:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773329.643648:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643650:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773329.643655:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c64a5fd1f80 x1862023308942848/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643662:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773329.643663:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773329.643665:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6493d12000. 02000000:00000001:0.0:1775773329.643668:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773329.643670:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773329.643672:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c649f468000 refcount=2 obd=lustre-OST0001-osc-ffff9c648a583800 00000020:00000001:0.0:1775773329.643674:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773329.643676:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c648b998000. 02000000:00000001:0.0:1775773329.643678:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773329.643679:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773329.643681:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c64a5fd1f80. 00000100:00000001:0.0:1775773329.643686:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773329.643687:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643689:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773329.643693:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c64a5fd0a80 x1862023308942592/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643700:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773329.643702:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773329.643704:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6493d13000. 02000000:00000001:0.0:1775773329.643707:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773329.643708:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773329.643710:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c64890b5800 refcount=5 obd=lustre-MDT0000-mdc-ffff9c648a583800 00000020:00000001:0.0:1775773329.643713:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773329.643715:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c648b99bc00. 02000000:00000001:0.0:1775773329.643718:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773329.643719:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773329.643720:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c64a5fd0a80. 00000100:00000001:0.0:1775773329.643723:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773329.643724:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643732:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773329.643737:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c64a5fd0e00 x1862023308942464/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773352 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773329.643744:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773329.643746:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773329.643748:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6493d11000. 02000000:00000001:0.0:1775773329.643751:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773329.643753:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773329.643754:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c6488034000 refcount=3 obd=MGC192.168.201.105@tcp 00000020:00000001:0.0:1775773329.643756:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773329.643758:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c648b99a000. 02000000:00000001:0.0:1775773329.643762:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773329.643764:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773329.643765:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c64a5fd0e00. 00000100:00000001:0.0:1775773329.643769:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773329.643771:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643774:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773329.643783:0:2357:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775773329.643785:0:2357:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773329.643788:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775773329.643789:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773329.771224:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773329.771230:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773329.771236:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773329.771237:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775773329.771283:0:193621:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775773329.771287:0:193621:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775773329.771291:0:193621:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:1.0:1775773330.007769:0:216843:0:(llite_lib.c:4054:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773330.794345:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773330.794353:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773330.794361:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773330.794363:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773330.794448:0:181745:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773330.794453:0:181745:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773330.794456:0:181745:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1775773331.818259:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773331.818266:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773331.818272:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773331.818273:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775773331.818343:0:190496:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775773331.818391:0:190496:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775773331.818394:0:190496:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775773332.202266:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) 89584a75-b38e-4303-b812-ccf0f9d4c217->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773332.202288:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1775773332.202290:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773332.202293:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c6488034000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773332.202297:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773332.202303:0:96535:0:(import.c:509:import_select_connection()) MGC192.168.201.105@tcp: connect to NID 192.168.201.105@tcp last attempt 9608 00000400:00000200:2.0:1775773332.202311:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773332.202318:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773332.202321:0:96535:0:(import.c:588:import_select_connection()) MGC192.168.201.105@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773332.202325:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.202329:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.202331:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.202334:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:2.0:1775773332.202337:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773332.202340:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56c88 00000020:00000001:2.0:1775773332.202343:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773332.203855:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c6488034800 refcount=3 00000020:00000001:2.0:1775773332.203862:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554325420032 : -109519384131584 : ffff9c6488034800) 00000020:00000001:2.0:1775773332.203867:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554325420032 : -109519384131584 : ffff9c6488034800) 00000100:00000040:2.0:1775773332.203874:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.203877:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.203880:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.203882:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:2.0:1775773332.203885:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c6488034800 : new refcount 2 00000100:00080000:2.0:1775773332.203889:0:96535:0:(import.c:623:import_select_connection()) MGC192.168.201.105@tcp: import ffff9c6488034000 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:2.0:1775773332.203892:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773332.203896:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773332.203901:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773332.203904:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.203907:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000100:00000001:2.0:1775773332.203910:0:96535:0:(obd_class.h:897:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773332.203919:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c648ac1d880. 00000020:00000040:2.0:1775773332.203936:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c6488034000 refcount=4 obd=MGC192.168.201.105@tcp 02000000:00000001:2.0:1775773332.203941:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773332.203943:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.203946:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773332.203949:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773332.203955:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773332.203964:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c6493d13000. 00000100:00000001:2.0:1775773332.203971:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773332.203982:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773332.203993:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773332.203999:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773332.204004:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000003f5cf362] to pc [ptlrpcd_rcv+-1] req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773332.204040:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773332.204045:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) d4b1039f-ed2d-40d9-b26a-50f3c64e108a->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773332.204049:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1775773332.204051:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773332.204053:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c64890b5800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773332.204058:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773332.204064:0:96535:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9c648a583800: connect to NID 192.168.201.105@tcp last attempt 9608 00000400:00000200:2.0:1775773332.204074:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773332.204079:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773332.204082:0:96535:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9c648a583800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773332.204085:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204088:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.204090:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204092:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:2.0:1775773332.204095:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773332.204097:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56cd5 00000020:00000001:2.0:1775773332.204099:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773332.204109:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c6484f70000 refcount=5 00000020:00000001:2.0:1775773332.204111:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554274283520 : -109519435268096 : ffff9c6484f70000) 00000020:00000001:2.0:1775773332.204113:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554274283520 : -109519435268096 : ffff9c6484f70000) 00000100:00000040:2.0:1775773332.204116:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204119:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.204122:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204124:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:2.0:1775773332.204127:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c6484f70000 : new refcount 4 00000100:00080000:2.0:1775773332.204131:0:96535:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9c648a583800: import ffff9c64890b5800 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204133:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773332.204136:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773332.204143:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773332.204146:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204148:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773332.204152:0:96535:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773332.204155:0:96535:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204157:0:96535:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773332.204186:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c648ac1d180. 00000020:00000040:2.0:1775773332.204190:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c64890b5800 refcount=6 obd=lustre-MDT0000-mdc-ffff9c648a583800 02000000:00000001:2.0:1775773332.204193:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773332.204194:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204196:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773332.204198:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773332.204211:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773332.204218:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c6493d11c00. 00000100:00000001:2.0:1775773332.204222:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773332.204229:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773332.204238:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773332.204243:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773332.204248:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000041ac3989] to pc [ptlrpcd_rcv+-1] req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773332.204270:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773332.204275:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) d4b1039f-ed2d-40d9-b26a-50f3c64e108a->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773332.204279:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1775773332.204288:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773332.204290:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c64890b0000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773332.204294:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773332.204297:0:96535:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9c648a583800: connect to NID 192.168.201.105@tcp last attempt 9608 00000400:00000200:2.0:1775773332.204302:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773332.204305:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773332.204307:0:96535:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9c648a583800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773332.204310:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204312:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.204314:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204316:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:2.0:1775773332.204318:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773332.204319:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56ce3 00000020:00000001:2.0:1775773332.204321:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773332.204323:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c649f46a000 refcount=4 00000020:00000001:2.0:1775773332.204325:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554715709440 : -109518993842176 : ffff9c649f46a000) 00000020:00000001:2.0:1775773332.204327:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554715709440 : -109518993842176 : ffff9c649f46a000) 00000100:00000040:2.0:1775773332.204330:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204332:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.204334:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204336:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:2.0:1775773332.204338:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c649f46a000 : new refcount 3 00000100:00080000:2.0:1775773332.204341:0:96535:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9c648a583800: import ffff9c64890b0000 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204343:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773332.204352:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773332.204356:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773332.204358:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204360:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773332.204364:0:96535:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773332.204367:0:96535:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204369:0:96535:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773332.204373:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c648ac1dc00. 00000020:00000040:2.0:1775773332.204377:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c64890b0000 refcount=3 obd=lustre-OST0000-osc-ffff9c648a583800 02000000:00000001:2.0:1775773332.204380:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773332.204381:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204382:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773332.204385:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773332.204387:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773332.204392:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c6493d11400. 00000100:00000001:2.0:1775773332.204396:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773332.204401:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773332.204408:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773332.204410:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773332.204415:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000817e592e] to pc [ptlrpcd_rcv+-1] req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773332.204425:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1775773332.204428:0:96535:0:(pinger.c:229:ptlrpc_pinger_process_import()) d4b1039f-ed2d-40d9-b26a-50f3c64e108a->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775773332.204432:0:96535:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1775773332.204433:0:96535:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1775773332.204435:0:96535:0:(import.c:65:import_set_state_nolock()) ffff9c649f468000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1775773332.204437:0:96535:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1775773332.204440:0:96535:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9c648a583800: connect to NID 192.168.201.105@tcp last attempt 9608 00000400:00000200:2.0:1775773332.204445:0:96535:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:2.0:1775773332.204450:0:96535:0:(peer.c:1530:LNetPeerDiscovered()) Peer NID 192.168.201.105@tcp is discovered: rc = 1 00000100:00080000:2.0:1775773332.204452:0:96535:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9c648a583800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1775773332.204463:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204466:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.204470:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204473:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000001:2.0:1775773332.204475:0:96535:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1775773332.204477:0:96535:0:(genops.c:823:class_conn2export()) looking for export cookie 0x1ad9b7fbd0c56cea 00000020:00000001:2.0:1775773332.204479:0:96535:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1775773332.204481:0:96535:0:(lustre_handles.c:131:class_handle2object()) GET export ffff9c649f46f000 refcount=4 00000020:00000001:2.0:1775773332.204483:0:96535:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446634554715729920 : -109518993821696 : ffff9c649f46f000) 00000020:00000001:2.0:1775773332.204484:0:96535:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446634554715729920 : -109518993821696 : ffff9c649f46f000) 00000100:00000040:2.0:1775773332.204488:0:96535:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff9c6485369f00 refcount 11 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204490:0:96535:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1775773332.204492:0:96535:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff9c6485369f00 refcount 12 to 192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204494:0:96535:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446634554278452992 : -109519431098624 : ffff9c6485369f00) 00000020:00000040:2.0:1775773332.204496:0:96535:0:(genops.c:898:class_export_put()) PUTting export ffff9c649f46f000 : new refcount 3 00000100:00080000:2.0:1775773332.204499:0:96535:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9c648a583800: import ffff9c649f468000 using connection 192.168.201.105@tcp/192.168.201.105@tcp 00000100:00000001:2.0:1775773332.204501:0:96535:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1775773332.204503:0:96535:0:(sec.c:1588:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1775773332.204507:0:96535:0:(sec.c:1628:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1775773332.204509:0:96535:0:(sec.c:1656:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204512:0:96535:0:(obd_class.h:890:obd_reconnect()) Process entered 00000008:00100000:2.0:1775773332.204514:0:96535:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1775773332.204516:0:96535:0:(osc_request.c:3832:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204517:0:96535:0:(obd_class.h:904:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1775773332.204521:0:96535:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9c648ac1ca80. 00000020:00000040:2.0:1775773332.204524:0:96535:0:(genops.c:1229:class_import_get()) import ffff9c649f468000 refcount=3 obd=lustre-OST0001-osc-ffff9c648a583800 02000000:00000001:2.0:1775773332.204527:0:96535:0:(sec.c:2064:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1775773332.204528:0:96535:0:(sec.c:2074:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1775773332.204529:0:96535:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1775773332.204531:0:96535:0:(sec.c:443:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1775773332.204532:0:96535:0:(sec.c:467:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1775773332.204536:0:96535:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff9c6493d12000. 00000100:00000001:2.0:1775773332.204565:0:96535:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1775773332.204572:0:96535:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1775773332.204580:0:96535:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1775773332.204582:0:96535:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773332.204586:0:96535:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f23f7190] to pc [ptlrpcd_rcv+-1] req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1775773332.204595:0:96535:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1775773332.204601:0:96535:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (9618) 00000100:00000001:0.0:1775773332.204822:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775773332.204826:0:2357:0:(client.c:1975:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1775773332.204829:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775773332.204835:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.204856:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775773332.204858:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.204862:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775773332.204864:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775773332.204868:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775773332.204872:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c648ac1d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:89584a75-b38e-4303-b812-ccf0f9d4c217:2357:1862023308942976:192.168.201.105@tcp:250:kworker.0 00000100:00000001:0.0:1775773332.204876:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775773332.204880:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775773332.204883:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.204886:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775773332.204891:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6490760c00. 02000000:00000001:0.0:1775773332.204893:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775773332.204898:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000010:0.0:1775773332.204903:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c64b2e6ff78. 00000400:00000010:0.0:1775773332.204909:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace25d8. 00000100:00000200:0.0:1775773332.204917:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308942976, portal 25 00000100:00000001:0.0:1775773332.204919:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775773332.204921:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554371463296 : -109519338088320 : ffff9c648ac1d880) 00000100:00000040:0.0:1775773332.204926:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.204931:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775773332.204933:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:0.0:1775773332.204936:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace23b8. 00000100:00000200:0.0:1775773332.204938:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1862023308942976, offset 0 00000400:00000200:0.0:1775773332.204942:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:0.0:1775773332.204949:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.204953:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000400:00000200:0.0:1775773332.204956:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775773332.204960:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000400:00000200:0.0:1775773332.204963:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775773332.204966:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000400:00000200:0.0:1775773332.204973:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337739:337739:256:4294967295] 192.168.201.105@tcp LPNI seq info [337684:337684:8:4294967295] 00000400:00000200:0.0:1775773332.204977:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.204984:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000800:00000200:0.0:1775773332.204989:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.204993:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c63e4eed900. 00000800:00000200:0.0:1775773332.204999:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.0:1775773332.205001:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000200:0.1F:1775773332.205005:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.1:1775773332.205011:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1775773332.205014:0:2357:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205016:0:2357:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c63e4eed900. 00000400:00000200:0.0:1775773332.205022:0:2357:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775773332.205025:0:2357:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775773332.205029:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace23b8 00000400:00000010:0.0:1775773332.205031:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace23b8. 00000100:00000001:0.0:1775773332.205034:0:2357:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775773332.205037:0:2357:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205043:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205046:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205051:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205052:0:2357:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775773332.205057:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775773332.205059:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773332.205061:0:2357:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS new deadline estimation for cpu 0 is 4304281993 00000100:00000010:0.0:1775773332.205064:0:2357:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c649f6d1180. 00000100:00000001:0.0:1775773332.205070:0:2357:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1775773332.205074:0:2357:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS new deadline estimation for cpu 1 is 4304281993 00000100:00000010:0.0:1775773332.205076:0:2357:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9c649f6d1300. 00000100:00000001:0.0:1775773332.205078:0:2357:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.201.105@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1775773332.205080:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775773332.205085:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205087:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773332.205090:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773332/real 1775773332] req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773332.205097:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773332.205098:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773332.205100:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c64b2e6ff78. 00000400:00000200:0.0:1775773332.205101:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace25d8 00000400:00000010:0.0:1775773332.205102:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace25d8. 00000100:00000001:0.0:1775773332.205104:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773332.205106:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773332.205112:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205116:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773332.205117:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205119:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205120:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773332.205122:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205126:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773332.205129:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205134:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773332.205135:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205136:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205137:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205139:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773332.205140:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773332.205142:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c6488034000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773332.205144:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773332.205149:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773332.205152:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773332.205158:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c648ac1d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:89584a75-b38e-4303-b812-ccf0f9d4c217:2357:1862023308942976:192.168.201.105@tcp:250:kworker.0 00000100:00000001:0.0:1775773332.205161:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775773332.205191:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205196:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775773332.205198:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.205199:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775773332.205202:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775773332.205204:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775773332.205212:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c648ac1d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308943104:192.168.201.105@tcp:38:kworker.0 00000100:00000001:0.0:1775773332.205215:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775773332.205221:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775773332.205222:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.205224:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775773332.205228:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6490761800. 02000000:00000001:0.0:1775773332.205230:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775773332.205234:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000010:0.0:1775773332.205236:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c64b2e6ff78. 00000400:00000010:0.0:1775773332.205240:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace25d8. 00000100:00000200:0.0:1775773332.205243:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308943104, portal 10 00000100:00000001:0.0:1775773332.205245:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775773332.205247:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554371461504 : -109519338090112 : ffff9c648ac1d180) 00000100:00000040:0.0:1775773332.205254:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205261:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775773332.205263:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:0.0:1775773332.205265:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace23b8. 00000100:00000200:0.0:1775773332.205271:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1862023308943104, offset 0 00000400:00000200:0.0:1775773332.205274:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:0.0:1775773332.205277:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.205280:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000400:00000200:0.0:1775773332.205282:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775773332.205284:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000400:00000200:0.0:1775773332.205286:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775773332.205287:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000400:00000200:0.0:1775773332.205290:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337740:337740:256:4294967295] 192.168.201.105@tcp LPNI seq info [337685:337685:8:4294967295] 00000400:00000200:0.0:1775773332.205293:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.205297:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000800:00000200:0.0:1775773332.205300:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205303:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c63e4eed900. 00000800:00000200:0.0:1775773332.205306:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.0:1775773332.205308:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000200:0.1:1775773332.205311:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.1:1775773332.205313:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1775773332.205316:0:2357:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205317:0:2357:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c63e4eed900. 00000400:00000200:0.0:1775773332.205319:0:2357:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775773332.205321:0:2357:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775773332.205323:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace23b8 00000400:00000010:0.0:1775773332.205324:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace23b8. 00000100:00000001:0.0:1775773332.205326:0:2357:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775773332.205329:0:2357:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205334:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205336:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205340:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205341:0:2357:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775773332.205343:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775773332.205349:0:2357:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: using a timeout of 33957 usecs (34 jiffies) 00000400:00000200:0.0:1775773332.205352:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773332.205353:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS new deadline estimation for cpu 0 is 4304281125 00000100:00000001:0.0:1775773332.205356:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS request active for cpu 0, keep current deadline 4304281993 00000100:00000001:0.0:1775773332.205357:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS new deadline estimation for cpu 1 is 4304281125 00000100:00000001:0.0:1775773332.205359:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff9c648a583800: PM QoS request active for cpu 1, keep current deadline 4304281993 00000100:00000001:0.0:1775773332.205360:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775773332.205361:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205363:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773332.205365:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773332/real 1775773332] req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773332.205370:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773332.205371:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773332.205372:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c64b2e6ff78. 00000400:00000200:0.0:1775773332.205374:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace25d8 00000400:00000010:0.0:1775773332.205375:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace25d8. 00000100:00000001:0.0:1775773332.205376:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773332.205378:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773332.205383:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205387:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773332.205388:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205389:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205390:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773332.205392:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205396:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773332.205398:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205406:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773332.205407:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205409:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205409:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205411:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773332.205411:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773332.205413:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c64890b5800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773332.205415:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773332.205417:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773332.205419:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773332.205424:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c648ac1d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308943104:192.168.201.105@tcp:38:kworker.0 00000100:00000001:0.0:1775773332.205426:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775773332.205428:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205432:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775773332.205433:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.205435:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775773332.205436:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775773332.205437:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775773332.205439:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c648ac1dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308943232:192.168.201.105@tcp:8:kworker.0 00000100:00000001:0.0:1775773332.205441:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775773332.205442:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775773332.205443:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.205444:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775773332.205446:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6490762400. 02000000:00000001:0.0:1775773332.205448:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775773332.205450:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000010:0.0:1775773332.205455:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c64b2e6ff78. 00000400:00000010:0.0:1775773332.205458:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace25d8. 00000100:00000200:0.0:1775773332.205460:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308943232, portal 4 00000100:00000001:0.0:1775773332.205461:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775773332.205462:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554371464192 : -109519338087424 : ffff9c648ac1dc00) 00000100:00000040:0.0:1775773332.205466:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205471:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775773332.205472:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:0.0:1775773332.205474:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace23b8. 00000100:00000200:0.0:1775773332.205475:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862023308943232, offset 0 00000400:00000200:0.0:1775773332.205477:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:0.0:1775773332.205480:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.205482:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000400:00000200:0.0:1775773332.205484:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775773332.205485:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000400:00000200:0.0:1775773332.205487:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775773332.205488:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000400:00000200:0.0:1775773332.205491:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337741:337741:256:4294967295] 192.168.201.105@tcp LPNI seq info [337686:337686:8:4294967295] 00000400:00000200:0.0:1775773332.205493:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.205497:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000800:00000200:0.0:1775773332.205500:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205502:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c63e4eed900. 00000800:00000200:0.0:1775773332.205505:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.0:1775773332.205506:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000200:0.1:1775773332.205509:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.1:1775773332.205511:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1775773332.205513:0:2357:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205515:0:2357:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c63e4eed900. 00000400:00000200:0.0:1775773332.205517:0:2357:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775773332.205519:0:2357:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775773332.205521:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace23b8 00000400:00000010:0.0:1775773332.205522:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace23b8. 00000100:00000001:0.0:1775773332.205527:0:2357:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775773332.205529:0:2357:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205534:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205536:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205557:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205559:0:2357:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775773332.205563:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775773332.205566:0:2357:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: using a timeout of 161587 usecs (162 jiffies) 00000400:00000200:0.0:1775773332.205570:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773332.205571:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 0 is 4304281253 00000100:00000001:0.0:1775773332.205574:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS request active for cpu 0, keep current deadline 4304281993 00000100:00000001:0.0:1775773332.205581:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 1 is 4304281253 00000100:00000001:0.0:1775773332.205583:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff9c648a583800: PM QoS request active for cpu 1, keep current deadline 4304281993 00000100:00000001:0.0:1775773332.205585:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775773332.205587:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205589:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773332.205593:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773332/real 1775773332] req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773332.205601:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773332.205603:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773332.205604:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c64b2e6ff78. 00000400:00000200:0.0:1775773332.205607:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace25d8 00000400:00000010:0.0:1775773332.205609:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace25d8. 00000100:00000001:0.0:1775773332.205611:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773332.205614:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773332.205622:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205641:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773332.205643:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205645:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205647:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773332.205651:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205657:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773332.205661:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205668:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773332.205671:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205672:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205673:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205676:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773332.205677:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773332.205680:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c64890b0000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773332.205683:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773332.205686:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773332.205690:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773332.205696:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c648ac1dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308943232:192.168.201.105@tcp:8:kworker.0 00000100:00000001:0.0:1775773332.205699:0:2357:0:(client.c:1798:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1775773332.205701:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205705:0:2357:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1775773332.205706:0:2357:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.205712:0:2357:0:(sec.c:718:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1775773332.205713:0:2357:0:(sec.c:749:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1775773332.205715:0:2357:0:(sec.c:793:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1775773332.205717:0:2357:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff9c648ac1ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308943360:192.168.201.105@tcp:8:kworker.0 00000100:00000001:0.0:1775773332.205720:0:2357:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1775773332.205721:0:2357:0:(sec.c:1089:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1775773332.205722:0:2357:0:(sec.c:1126:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1775773332.205723:0:2357:0:(sec.c:1873:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1775773332.205725:0:2357:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff9c6490763400. 02000000:00000001:0.0:1775773332.205727:0:2357:0:(sec.c:1883:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1775773332.205729:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000010:0.0:1775773332.205731:0:2357:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff9c64b2e6ff78. 00000400:00000010:0.0:1775773332.205733:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace25d8. 00000100:00000200:0.0:1775773332.205735:0:2357:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1862023308943360, portal 4 00000100:00000001:0.0:1775773332.205737:0:2357:0:(client.c:3245:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1775773332.205738:0:2357:0:(client.c:3247:ptlrpc_request_addref()) Process leaving (rc=18446634554371459712 : -109519338091904 : ffff9c648ac1ca80) 00000100:00000040:0.0:1775773332.205767:0:2357:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205771:0:2357:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1775773332.205773:0:2357:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.105@tcp 00000400:00000010:0.0:1775773332.205775:0:2357:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9c648ace23b8. 00000100:00000200:0.0:1775773332.205776:0:2357:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1862023308943360, offset 0 00000400:00000200:0.0:1775773332.205779:0:2357:0:(lib-move.c:5324:LNetPut()) LNetPut -> 12345-192.168.201.105@tcp 00000400:00000200:0.0:1775773332.205782:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.205784:0:2357:0:(lib-move.c:2975:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.105@tcp local destination 00000400:00000200:0.0:1775773332.205786:0:2357:0:(lib-move.c:2605:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1775773332.205788:0:2357:0:(lib-move.c:1791:lnet_get_best_ni()) selected best_ni 192.168.201.5@tcp 00000400:00000200:0.0:1775773332.205790:0:2357:0:(lib-move.c:1345:lnet_select_peer_ni()) 192.168.201.5@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1775773332.205791:0:2357:0:(lib-move.c:1414:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.105@tcp 00000400:00000200:0.0:1775773332.205794:0:2357:0:(lib-move.c:1911:lnet_handle_send()) 192.168.201.5@tcp NI seq info: [337742:337742:256:4294967295] 192.168.201.105@tcp LPNI seq info [337687:337687:8:4294967295] 00000400:00000200:0.0:1775773332.205796:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000400:00000200:0.0:1775773332.205801:0:2357:0:(lib-move.c:2026:lnet_handle_send()) TRACE: 192.168.201.5@tcp(192.168.201.5@tcp:) -> 192.168.201.105@tcp(192.168.201.105@tcp:192.168.201.105@tcp) : PUT try# 0 00000800:00000200:0.0:1775773332.205808:0:2357:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205810:0:2357:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9c63e4eed900. 00000800:00000200:0.0:1775773332.205813:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.0:1775773332.205815:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000200:0.1:1775773332.205818:0:2357:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c6493d09480] -> 12345-192.168.201.105@tcp (2) 00000800:00000200:0.1:1775773332.205819:0:2357:0:(socklnd_cb.c:848:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.105 (cnted 0, interval 4, 2 secs later) 00000800:00000100:0.0:1775773332.205822:0:2357:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.105@tcp 00000800:00000010:0.0:1775773332.205823:0:2357:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9c63e4eed900. 00000400:00000200:0.0:1775773332.205826:0:2357:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1775773332.205828:0:2357:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.201.5@tcp->192.168.201.105@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1775773332.205830:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace23b8 00000400:00000010:0.0:1775773332.205831:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace23b8. 00000100:00000001:0.0:1775773332.205833:0:2357:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1775773332.205835:0:2357:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205840:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205842:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 0 dl 1775773357 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205847:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205848:0:2357:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1775773332.205850:0:2357:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775773332.205852:0:2357:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: using a timeout of 170937 usecs (171 jiffies) 00000400:00000200:0.0:1775773332.205854:0:2357:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.201.105@tcp to cpt 0 00000100:00000001:0.0:1775773332.205855:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 0 is 4304281262 00000100:00000001:0.0:1775773332.205857:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS request active for cpu 0, keep current deadline 4304281993 00000100:00000001:0.0:1775773332.205859:0:2357:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS new deadline estimation for cpu 1 is 4304281262 00000100:00000001:0.0:1775773332.205860:0:2357:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0001-osc-ffff9c648a583800: PM QoS request active for cpu 1, keep current deadline 4304281993 00000100:00000001:0.0:1775773332.205862:0:2357:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1775773332.205863:0:2357:0:(client.c:1926:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.205864:0:2357:0:(client.c:2461:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1775773332.205867:0:2357:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775773332/real 1775773332] req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1775773332.205889:0:2357:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1775773332.205890:0:2357:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1775773332.205891:0:2357:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff9c64b2e6ff78. 00000400:00000200:0.0:1775773332.205893:0:2357:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9c648ace25d8 00000400:00000010:0.0:1775773332.205894:0:2357:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff9c648ace25d8. 00000100:00000001:0.0:1775773332.205895:0:2357:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1775773332.205898:0:2357:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1775773332.205904:0:2357:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205908:0:2357:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1775773332.205909:0:2357:0:(client.c:2958:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205910:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205911:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1775773332.205913:0:2357:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205917:0:2357:0:(client.c:2515:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1775773332.205919:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205923:0:2357:0:(client.c:2109:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1775773332.205924:0:2357:0:(client.c:2950:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205925:0:2357:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1775773332.205926:0:2357:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205927:0:2357:0:(import.c:1060:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1775773332.205928:0:2357:0:(import.c:1091:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1775773332.205929:0:2357:0:(import.c:65:import_set_state_nolock()) ffff9c649f468000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775773332.205931:0:2357:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.105@tcp failed (-110) 00000100:00000001:0.0:1775773332.205932:0:2357:0:(import.c:1512:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1775773332.205939:0:2357:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775773332.205944:0:2357:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff9c648ac1ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:d4b1039f-ed2d-40d9-b26a-50f3c64e108a:2357:1862023308943360:192.168.201.105@tcp:8:kworker.0 00000100:00000001:0.0:1775773332.205946:0:2357:0:(client.c:2441:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205948:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205949:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c648ac1d880 x1862023308942976/t0(0) o250->MGC192.168.201.105@tcp@192.168.201.105@tcp:26/25 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205954:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773332.205955:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773332.205957:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6490760c00. 02000000:00000001:0.0:1775773332.205959:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773332.205960:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773332.205961:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c6488034000 refcount=3 obd=MGC192.168.201.105@tcp 00000020:00000001:0.0:1775773332.205963:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773332.205965:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c6493d13000. 02000000:00000001:0.0:1775773332.205968:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773332.205969:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773332.205970:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c648ac1d880. 00000100:00000001:0.0:1775773332.205973:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773332.205973:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205975:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205976:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c648ac1d180 x1862023308943104/t0(0) o38->lustre-MDT0000-mdc-ffff9c648a583800@192.168.201.105@tcp:12/10 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.205980:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773332.205981:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773332.205982:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6490761800. 02000000:00000001:0.0:1775773332.205983:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773332.205984:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773332.205985:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c64890b5800 refcount=5 obd=lustre-MDT0000-mdc-ffff9c648a583800 00000020:00000001:0.0:1775773332.205986:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773332.205986:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c6493d11c00. 02000000:00000001:0.0:1775773332.205988:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773332.205988:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773332.205992:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c648ac1d180. 00000100:00000001:0.0:1775773332.205994:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773332.205994:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.205996:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.205998:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c648ac1dc00 x1862023308943232/t0(0) o8->lustre-OST0000-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.206002:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773332.206002:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773332.206003:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6490762400. 02000000:00000001:0.0:1775773332.206005:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773332.206005:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773332.206006:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c64890b0000 refcount=2 obd=lustre-OST0000-osc-ffff9c648a583800 00000020:00000001:0.0:1775773332.206007:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773332.206008:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c6493d11400. 02000000:00000001:0.0:1775773332.206009:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773332.206010:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773332.206011:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c648ac1dc00. 00000100:00000001:0.0:1775773332.206012:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773332.206012:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.206013:0:2357:0:(client.c:2843:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1775773332.206015:0:2357:0:(client.c:2851:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff9c648ac1ca80 x1862023308943360/t0(0) o8->lustre-OST0001-osc-ffff9c648a583800@192.168.201.105@tcp:28/4 lens 520/544 e 0 to 1 dl 1775773357 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1775773332.206019:0:2357:0:(client.c:2769:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1775773332.206019:0:2357:0:(sec.c:1898:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1775773332.206020:0:2357:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff9c6490763400. 02000000:00000001:0.0:1775773332.206021:0:2357:0:(sec.c:1912:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1775773332.206022:0:2357:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1775773332.206023:0:2357:0:(genops.c:1242:class_import_put()) import ffff9c649f468000 refcount=2 obd=lustre-OST0001-osc-ffff9c648a583800 00000020:00000001:0.0:1775773332.206024:0:2357:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1775773332.206025:0:2357:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff9c6493d12000. 02000000:00000001:0.0:1775773332.206026:0:2357:0:(sec.c:483:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1775773332.206026:0:2357:0:(sec.c:500:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1775773332.206027:0:2357:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9c648ac1ca80. 00000100:00000001:0.0:1775773332.206029:0:2357:0:(client.c:2825:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1775773332.206030:0:2357:0:(client.c:2884:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.206031:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1775773332.206039:0:2357:0:(client.c:2615:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1775773332.206041:0:2357:0:(client.c:2643:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1775773332.206042:0:2357:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1775773332.206043:0:2357:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773332.842220:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773332.842224:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773332.842227:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773332.842229:0:160383:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775773332.842231:0:160383:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775773332.842236:0:160383:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1775773332.842237:0:160383:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1775773332.842239:0:160383:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773332.842241:0:160383:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775773332.842243:0:160383:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773332.842244:0:160383:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773332.842246:0:160383:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775773332.842288:0:181746:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775773332.842291:0:181746:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775773332.842294:0:181746:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1775773333.137235:0:216980:0:(llite_lib.c:4054:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1775773333.162212:0:160383:0:(connection.c:75:cpu_latency_work()) work item of ffff9c648a301800 (cpu 0) has reached its deadline 4304281993, at 4304282048 00000100:00000010:0.0:1775773333.162221:0:160383:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c649f6d1180. 00000100:00000040:1.0:1775773333.163207:0:174851:0:(connection.c:75:cpu_latency_work()) work item of ffff9c648a301910 (cpu 0) has reached its deadline 4304281993, at 4304282049 00000100:00000010:1.0:1775773333.163214:0:174851:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9c649f6d1300. 00000001:00000001:2.0:1775773333.254032:0:217016:0:(debug.c:644:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:2.0:1775773333.254037:0:217016:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity test_812b: @@@@@@ FAIL: client: import is not in IDLE state after 50 00000001:00000001:2.0:1775773333.256821:0:217016:0:(debug.c:647:libcfs_debug_mark_buffer()) ************************************************** 00010000:00000001:0.0:1775773333.866260:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773333.866266:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773333.866272:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773333.866273:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1775773333.866346:0:193621:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1775773333.866350:0:193621:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1775773333.866354:0:193621:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1775773334.890303:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773334.890309:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773334.890315:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773334.890316:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1775773334.890378:0:181745:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1775773334.890381:0:181745:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1775773334.890384:0:181745:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1775773335.914230:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773335.914236:0:160383:0:(ldlm_request.c:2313:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1775773335.914238:0:160383:0:(ldlm_request.c:2101:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1775773335.914241:0:160383:0:(ldlm_request.c:2273:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773335.914245:0:160383:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1775773335.914248:0:160383:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773335.914250:0:160383:0:(ldlm_request.c:2323:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773335.914253:0:160383:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1775773335.914257:0:160383:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1775773335.914259:0:160383:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1775773335.914314:0:190496:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1775773335.914318:0:190496:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1775773335.914321:0:190496:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1775773336.170245:0:2349:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9c648a5f1180] -> 12345-192.168.201.5@tcp (6) 00000800:00000200:3.0:1775773336.170251:0:2349:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.5@tcp ip 192.168.201.5:988 00000800:00000200:3.0:1775773336.170255:0:2349:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff9c6488ea1b00 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:1.0:1775773336.170392:0:2350:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000001:1.0:1775773336.170398:0:2350:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775773336.170407:0:2350:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1775773336.170412:0:2350:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1775773336.170414:0:2350:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1775773336.255551:0:217115:0:(llite_lib.c:4054:ll_show_options()) Process leaving (rc=0 : 0 : 0) Debug log: 1353 lines, 1353 kept, 0 dropped, 0 bad.