00000100:00000001:0.0F:1748873789.452669:0:6417:0:(client.c:2515:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1748873789.452681:0:6417:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1748873789.452682:0:6417:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873789.452684:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1748873789.452686:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1748873789.452688:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873789.452689:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1748873789.807757:0:24853:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1748873789.807763:0:24853:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1748873789.807771:0:24853:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1748873789.807773:0:24853:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0F:1748873789.807889:0:16899:0:(genops.c:1912:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1748873789.807906:0:16899:0:(genops.c:1926:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1748873789.807911:0:16899:0:(ldlm_lockd.c:2816:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1748873789.864466:0:5644:0:(llite_lib.c:4119:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.452710:0:6417:0:(client.c:2515:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1748873790.452716:0:6417:0:(client.c:2435:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1748873790.452731:0:6417:0:(client.c:2451:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1748873765/real 1748873765] req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1748873790.452749:0:6417:0:(lib-msg.c:21:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1748873790.452751:0:6417:0:(lib-msg.c:30:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1748873790.452754:0:6417:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8800b6e440b0. 00000400:00000200:0.0:1748873790.452758:0:6417:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800aa3b9220 00000400:00000010:0.0:1748873790.452760:0:6417:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800aa3b9220. 00000100:00000001:0.0:1748873790.452764:0:6417:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000200:0.0:1748873790.452767:0:6417:0:(events.c:81:reply_in_callback()) @@@ type 6, status 0 req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1748873790.452777:0:6417:0:(events.c:103:reply_in_callback()) @@@ unlink req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.452809:0:6417:0:(events.c:164:reply_in_callback()) Process leaving 00000100:00000001:0.0:1748873790.452811:0:6417:0:(client.c:2932:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.452814:0:6417:0:(niobuf.c:441:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1748873790.452817:0:6417:0:(niobuf.c:455:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1748873790.452820:0:6417:0:(client.c:2482:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.452829:0:6417:0:(client.c:2487:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.452837:0:6417:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1748873790.452838:0:6417:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.452849:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1748873790.452851:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1748873790.452854:0:6417:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1748873790.452858:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.452867:0:6417:0:(client.c:2097:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1748873790.452869:0:6417:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.452871:0:6417:0:(niobuf.c:441:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1748873790.452872:0:6417:0:(niobuf.c:455:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.452874:0:6417:0:(import.c:1037:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1748873790.452877:0:6417:0:(import.c:1068:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1748873790.452880:0:6417:0:(import.c:66:import_set_state_nolock()) ffff8801373af000 lustre-OST2710_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1748873790.452908:0:6417:0:(import.c:1483:ptlrpc_connect_interpret()) recovery of lustre-OST2710_UUID on 192.168.202.156@tcp failed (-110) 00000100:00000001:0.0:1748873790.452911:0:6417:0:(import.c:1489:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1748873790.452914:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1748873790.452923:0:6417:0:(client.c:2365:ptlrpc_check_set()) Completed RPC req@ffff8801369c9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:106e5209-6aaf-48fa-ba19-a17064ac80d5:6417:1833823818219264:192.168.202.156@tcp:8: 00000100:00000001:0.0:1748873790.452927:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.452930:0:6417:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1748873790.452932:0:6417:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8801369c9180 x1833823818219264/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 1 dl 1748873790 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0F:1748873790.452939:0:5519:0:(pinger.c:229:ptlrpc_pinger_process_import()) 05a09d41-c85d-4993-a6db-87da69cb8def->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1748873790.452940:0:6417:0:(client.c:2741:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1748873790.452942:0:6417:0:(sec.c:1818:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1748873790.452946:0:6417:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8800a7401000. 02000000:00000001:0.0:1748873790.452950:0:6417:0:(sec.c:1832:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1748873790.452952:0:6417:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1748873790.452954:0:5519:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000020:00000040:0.0:1748873790.452954:0:6417:0:(genops.c:1242:class_import_put()) import ffff8801373af000 refcount=5 obd=lustre-OST2710-osc-ffff8800a82f5800 00000020:00000001:0.0:1748873790.452956:0:6417:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1748873790.452958:0:6417:0:(sec_null.c:173:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at ffff8800b6cea000. 00000100:00000010:2.0:1748873790.452960:0:5519:0:(client.c:585:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8801369ca300. 02000000:00000001:0.0:1748873790.452963:0:6417:0:(sec.c:468:sptlrpc_req_put_ctx()) Process entered 00000020:00000040:2.0:1748873790.452964:0:5519:0:(genops.c:1229:class_import_get()) import ffff8800b63a2800 refcount=4 obd=MGC192.168.202.156@tcp 00000100:00000001:2.0:1748873790.452968:0:5519:0:(client.c:837:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1748873790.452969:0:6417:0:(sec.c:485:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1748873790.452971:0:5519:0:(sec.c:430:sptlrpc_req_get_ctx()) Process entered 00000100:00000010:0.0:1748873790.452971:0:6417:0:(client.c:591:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8801369c9180. 00000100:00000001:0.0:1748873790.452974:0:6417:0:(client.c:2799:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1748873790.452975:0:6417:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1748873790.452976:0:5519:0:(sec.c:454:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.452977:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1748873790.452981:0:5519:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8800b6dcf900. 00000100:00000001:0.0:1748873790.452981:0:6417:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1748873790.452982:0:6417:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.452984:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1748873790.452985:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.452987:0:5519:0:(client.c:917:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1748873790.452995:0:5519:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 05a09d41-c85d-4993-a6db-87da69cb8def->MGS req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1748873790.453007:0:5519:0:(jobid.c:925:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1748873790.453010:0:5519:0:(jobid.c:936:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1748873790.453015:0:5519:0:(ptlrpcd.c:236:ptlrpcd_add_req()) @@@ add req [ffff8801369ca300] to pc [ptlrpcd_00_02+2] req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1748873790.453051:0:5519:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1748873790.453054:0:5519:0:(pinger.c:229:ptlrpc_pinger_process_import()) 106e5209-6aaf-48fa-ba19-a17064ac80d5->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 1 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1748873790.453058:0:5519:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1748873790.453060:0:5519:0:(import.c:693:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1748873790.453062:0:5519:0:(import.c:66:import_set_state_nolock()) ffff8800aa0cc000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1748873790.453065:0:5519:0:(import.c:480:import_select_connection()) Process entered 00000100:00000040:2.0:1748873790.453068:0:5519:0:(lustre_net.h:1992:ptlrpc_connection_put()) PUT conn=ffff88012d03aa20 refcount 14 to 192.168.202.156@tcp 00000100:00000001:0.0:1748873790.453069:0:6420:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1748873790.453071:0:5519:0:(connection.c:159:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1748873790.453072:0:6420:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1748873790.453073:0:5519:0:(connection.c:164:ptlrpc_connection_addref()) conn=ffff88012d03aa20 refcount 15 to 192.168.202.156@tcp 00000100:00000001:0.0:1748873790.453074:0:6420:0:(client.c:1768:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1748873790.453075:0:5519:0:(connection.c:166:ptlrpc_connection_addref()) Process leaving (rc=18446612137364400672 : -131936345150944 : ffff88012d03aa20) 00000020:00000001:2.0:1748873790.453078:0:5519:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0F:1748873790.453079:0:6421:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000020:00000040:2.0:1748873790.453080:0:5519:0:(genops.c:823:class_conn2export()) looking for export cookie 0x696e714cbd1dbc9a 00000100:00000040:0.0:1748873790.453080:0:6420:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1748873790.453082:0:5519:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1748873790.453085:0:5519:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8800a82f3000 refcount=5 00000020:00000001:2.0:1748873790.453087:0:5519:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446612135135883264 : -131938573668352 : ffff8800a82f3000) 00000020:00000001:2.0:1748873790.453089:0:5519:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446612135135883264 : -131938573668352 : ffff8800a82f3000) 00000100:00000001:0.0:1748873790.453089:0:6420:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1748873790.453090:0:6421:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1748873790.453090:0:6420:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1748873790.453092:0:5519:0:(lustre_net.h:1992:ptlrpc_connection_put()) PUT conn=ffff88012d03aa20 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1748873790.453093:0:6421:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1748873790.453093:0:6420:0:(sec.c:697:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1748873790.453096:0:6420:0:(sec.c:728:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1748873790.453099:0:5519:0:(connection.c:159:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1748873790.453099:0:6420:0:(sec.c:756:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1748873790.453101:0:5519:0:(connection.c:164:ptlrpc_connection_addref()) conn=ffff88012d03aa20 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1748873790.453102:0:5519:0:(connection.c:166:ptlrpc_connection_addref()) Process leaving (rc=18446612137364400672 : -131936345150944 : ffff88012d03aa20) 00000100:00100000:0.0:1748873790.453102:0:6420:0:(client.c:1874:ptlrpc_send_new_req()) Sending RPC req@ffff8801369ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:05a09d41-c85d-4993-a6db-87da69cb8def:6420:1833823818220928:192.168.202.156@tcp:400: 00000020:00000040:2.0:1748873790.453105:0:5519:0:(genops.c:898:class_export_put()) PUTting export ffff8800a82f3000 : new refcount 4 00000100:00000001:0.0:1748873790.453106:0:6420:0:(niobuf.c:817:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1748873790.453108:0:5519:0:(import.c:619:import_select_connection()) lustre-MDT0000-mdc-ffff8800a82f5800: import ffff8800aa0cc000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:2.0:1748873790.453110:0:5519:0:(import.c:623:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1748873790.453110:0:6420:0:(sec.c:1047:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1748873790.453112:0:5519:0:(sec.c:1538:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1748873790.453112:0:6421:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1748873790.453112:0:6420:0:(sec.c:1084:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453114:0:6421:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1748873790.453115:0:6420:0:(sec.c:1796:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1748873790.453116:0:6421:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1748873790.453118:0:5519:0:(sec.c:1578:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1748873790.453118:0:6421:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1748873790.453118:0:6420:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8800b3714c00. 02000000:00000001:2.0:1748873790.453120:0:5519:0:(sec.c:1606:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1748873790.453120:0:6420:0:(sec.c:1806:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.453122:0:5519:0:(obd_class.h:910:obd_reconnect()) Process entered 00000400:00000010:0.0:1748873790.453124:0:6420:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8800b6e440b0. 00000008:00100000:2.0:1748873790.453126:0:5519:0:(osc_request.c:3772:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34617127 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1748873790.453129:0:5519:0:(osc_request.c:3775:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1748873790.453129:0:6420:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800aa3b9220. 00000100:00000001:2.0:1748873790.453131:0:5519:0:(obd_class.h:924:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1748873790.453134:0:5519:0:(client.c:585:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8801369cb100. 00000100:00000200:0.0:1748873790.453134:0:6420:0:(niobuf.c:1048:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1833823818220928, portal 25 00000020:00000040:2.0:1748873790.453136:0:5519:0:(genops.c:1229:class_import_get()) import ffff8800aa0cc000 refcount=11 obd=lustre-MDT0000-mdc-ffff8800a82f5800 00000100:00000001:0.0:1748873790.453136:0:6420:0:(client.c:3219:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1748873790.453138:0:6420:0:(client.c:3221:ptlrpc_request_addref()) Process leaving (rc=18446612137525420800 : -131936184130816 : ffff8801369ca300) 02000000:00000001:2.0:1748873790.453139:0:5519:0:(sec.c:1984:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1748873790.453140:0:5519:0:(sec.c:1994:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.453143:0:5519:0:(client.c:837:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1748873790.453143:0:6420:0:(niobuf.c:1068:ptl_send_rpc()) @@@ send flags=200 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1748873790.453144:0:5519:0:(sec.c:430:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1748873790.453146:0:5519:0:(sec.c:454:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1748873790.453149:0:5519:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8800b6ceb400. 00000100:00000001:0.0:1748873790.453150:0:6420:0:(niobuf.c:50:ptl_send_buf()) Process entered 00000100:00000001:2.0:1748873790.453153:0:5519:0:(client.c:917:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1748873790.453154:0:6420:0:(niobuf.c:53:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00100000:2.0:1748873790.453158:0:5519:0:(import.c:818:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1748873790.453161:0:6420:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800aa3b92a8. 00000100:00000200:0.0:1748873790.453164:0:6420:0:(niobuf.c:81:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1833823818220928, offset 0 00000100:00000001:2.0:1748873790.453166:0:5519:0:(jobid.c:925:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1748873790.453167:0:5519:0:(jobid.c:936:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1748873790.453170:0:6420:0:(lib-move.c:5265:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000040:2.0:1748873790.453171:0:5519:0:(ptlrpcd.c:236:ptlrpcd_add_req()) @@@ add req [ffff8801369cb100] to pc [ptlrpcd_rcv+-1] req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1748873790.453182:0:6420:0:(lib-move.c:2934:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:0.0:1748873790.453188:0:6420:0:(lib-move.c:2564:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1748873790.453191:0:5519:0:(import.c:825:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1748873790.453192:0:6420:0:(lib-move.c:1750:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00080000:2.0:1748873790.453193:0:5519:0:(pinger.c:229:ptlrpc_pinger_process_import()) 106e5209-6aaf-48fa-ba19-a17064ac80d5->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1748873790.453194:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1748873790.453196:0:5519:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1748873790.453196:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1748873790.453196:0:6420:0:(lib-move.c:1329:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1748873790.453197:0:6417:0:(client.c:1768:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1748873790.453198:0:6420:0:(lib-move.c:1398:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000100:00000001:2.0:1748873790.453202:0:5519:0:(import.c:693:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1748873790.453203:0:5519:0:(import.c:66:import_set_state_nolock()) ffff8801373ab800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1748873790.453203:0:6420:0:(lib-move.c:1870:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [2050:2050:256:4294967295] 192.168.202.156@tcp LPNI seq info [2050:2050:8:4294967295] 00000100:00000001:2.0:1748873790.453205:0:5519:0:(import.c:480:import_select_connection()) Process entered 00000100:00000040:1.0:1748873790.453205:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1748873790.453208:0:5519:0:(lustre_net.h:1992:ptlrpc_connection_put()) PUT conn=ffff88012d03aa20 refcount 14 to 192.168.202.156@tcp 00000100:00000001:2.0:1748873790.453210:0:5519:0:(connection.c:159:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1748873790.453212:0:5519:0:(connection.c:164:ptlrpc_connection_addref()) conn=ffff88012d03aa20 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1748873790.453214:0:5519:0:(connection.c:166:ptlrpc_connection_addref()) Process leaving (rc=18446612137364400672 : -131936345150944 : ffff88012d03aa20) 00000100:00000001:1.0:1748873790.453215:0:6417:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1748873790.453215:0:6420:0:(lib-move.c:1985:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000020:00000001:2.0:1748873790.453216:0:5519:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:1.0:1748873790.453216:0:6417:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1748873790.453217:0:5519:0:(genops.c:823:class_conn2export()) looking for export cookie 0x696e714cbd1dbca1 00000020:00000001:2.0:1748873790.453219:0:5519:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:1.0:1748873790.453219:0:6417:0:(sec.c:697:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1748873790.453220:0:5519:0:(lustre_handles.c:131:class_handle2object()) GET export ffff8800a82f5000 refcount=5 00000800:00000200:0.0:1748873790.453220:0:6420:0:(socklnd_cb.c:1000:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.202.156@tcp 02000000:00000001:1.0:1748873790.453221:0:6417:0:(sec.c:728:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1748873790.453222:0:5519:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446612135135891456 : -131938573660160 : ffff8800a82f5000) 02000000:00000001:1.0:1748873790.453223:0:6417:0:(sec.c:756:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1748873790.453224:0:5519:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446612135135891456 : -131938573660160 : ffff8800a82f5000) 00000800:00000010:0.0:1748873790.453225:0:6420:0:(socklnd_cb.c:42:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8800b6f3a200. 00000100:00000040:2.0:1748873790.453227:0:5519:0:(lustre_net.h:1992:ptlrpc_connection_put()) PUT conn=ffff88012d03aa20 refcount 14 to 192.168.202.156@tcp 00000100:00100000:1.0:1748873790.453227:0:6417:0:(client.c:1874:ptlrpc_send_new_req()) Sending RPC req@ffff8801369cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:106e5209-6aaf-48fa-ba19-a17064ac80d5:6417:1833823818221056:192.168.202.156@tcp:38: 00000100:00000001:2.0:1748873790.453228:0:5519:0:(connection.c:159:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1748873790.453230:0:5519:0:(connection.c:164:ptlrpc_connection_addref()) conn=ffff88012d03aa20 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1748873790.453231:0:6417:0:(niobuf.c:817:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1748873790.453232:0:5519:0:(connection.c:166:ptlrpc_connection_addref()) Process leaving (rc=18446612137364400672 : -131936345150944 : ffff88012d03aa20) 00000800:00000200:0.0:1748873790.453232:0:6420:0:(socklnd.c:225:ksocknal_find_peer_locked()) got peer_ni [ffff8800b3e4fd80] -> 12345-192.168.202.156@tcp (5) 00000020:00000040:2.0:1748873790.453233:0:5519:0:(genops.c:898:class_export_put()) PUTting export ffff8800a82f5000 : new refcount 4 02000000:00000001:1.0:1748873790.453233:0:6417:0:(sec.c:1047:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1748873790.453235:0:6417:0:(sec.c:1084:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1748873790.453236:0:5519:0:(import.c:619:import_select_connection()) lustre-MDT0001-mdc-ffff8800a82f5800: import ffff8801373ab800 using connection 192.168.202.156@tcp/192.168.202.156@tcp 02000000:00000001:1.0:1748873790.453237:0:6417:0:(sec.c:1796:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1748873790.453238:0:5519:0:(import.c:623:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1748873790.453239:0:5519:0:(sec.c:1538:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1748873790.453240:0:6417:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8800ab5c1c00. 00000800:00000200:0.0:1748873790.453240:0:6420:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.156@tcp ip 192.168.202.156:988 02000000:00000001:2.0:1748873790.453242:0:5519:0:(sec.c:1578:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1748873790.453242:0:6417:0:(sec.c:1806:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1748873790.453244:0:5519:0:(sec.c:1606:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1748873790.453245:0:6420:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8800b6f3a200 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1748873790.453246:0:5519:0:(obd_class.h:910:obd_reconnect()) Process entered 00000400:00000010:1.0:1748873790.453246:0:6417:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8800aa014580. 00000008:00100000:2.0:1748873790.453247:0:5519:0:(osc_request.c:3772:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34617127 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1748873790.453249:0:5519:0:(osc_request.c:3775:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.453251:0:5519:0:(obd_class.h:924:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1748873790.453251:0:6417:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800ab5d3f68. 00000100:00000010:2.0:1748873790.453253:0:5519:0:(client.c:585:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8801369c8000. 00000020:00000040:2.0:1748873790.453255:0:5519:0:(genops.c:1229:class_import_get()) import ffff8801373ab800 refcount=6 obd=lustre-MDT0001-mdc-ffff8800a82f5800 00000100:00000200:1.0:1748873790.453255:0:6417:0:(niobuf.c:1048:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1833823818221056, portal 10 00000100:00000001:0.0:1748873790.453256:0:6420:0:(niobuf.c:98:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1748873790.453257:0:5519:0:(sec.c:1984:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1748873790.453257:0:6417:0:(client.c:3219:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1748873790.453258:0:5519:0:(sec.c:1994:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453258:0:6417:0:(client.c:3221:ptlrpc_request_addref()) Process leaving (rc=18446612137525424384 : -131936184127232 : ffff8801369cb100) 00000100:00000001:0.0:1748873790.453260:0:6420:0:(niobuf.c:626:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 0 is 4301309805 00000100:00000001:2.0:1748873790.453264:0:5519:0:(client.c:837:ptlrpc_request_bufs_pack()) Process entered 00000100:00000010:0.0:1748873790.453264:0:6420:0:(niobuf.c:629:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 56 at ffff8800b6ce2540. 02000000:00000001:2.0:1748873790.453265:0:5519:0:(sec.c:430:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1748873790.453265:0:6417:0:(niobuf.c:1068:ptl_send_rpc()) @@@ send flags=200 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1748873790.453266:0:5519:0:(sec.c:454:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1748873790.453269:0:5519:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8800b6ce9800. 00000100:00000001:0.0:1748873790.453270:0:6420:0:(niobuf.c:641:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1748873790.453273:0:5519:0:(client.c:917:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453273:0:6417:0:(niobuf.c:50:ptl_send_buf()) Process entered 00000100:00000001:0.0:1748873790.453275:0:6420:0:(niobuf.c:626:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 1 is 4301309805 00000100:00000040:1.0:1748873790.453276:0:6417:0:(niobuf.c:53:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00100000:2.0:1748873790.453277:0:5519:0:(import.c:818:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:0.0:1748873790.453277:0:6420:0:(niobuf.c:629:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 56 at ffff8800b6ce2880. 00000400:00000010:1.0:1748873790.453278:0:6417:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800ab5d3110. 00000100:00000001:0.0:1748873790.453279:0:6420:0:(niobuf.c:641:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 1 00000100:00000200:1.0:1748873790.453280:0:6417:0:(niobuf.c:81:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1833823818221056, offset 0 00000100:00000001:0.0:1748873790.453281:0:6420:0:(niobuf.c:626:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 2 is 4301309805 00000100:00000010:0.0:1748873790.453283:0:6420:0:(niobuf.c:629:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 56 at ffff8800b6ce24c0. 00000100:00000001:2.0:1748873790.453284:0:5519:0:(jobid.c:925:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1748873790.453285:0:5519:0:(jobid.c:936:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.453286:0:6420:0:(niobuf.c:641:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 2 00000100:00000040:2.0:1748873790.453288:0:5519:0:(ptlrpcd.c:236:ptlrpcd_add_req()) @@@ add req [ffff8801369c8000] to pc [ptlrpcd_rcv+-1] req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.453288:0:6420:0:(niobuf.c:626:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS new deadline estimation for cpu 3 is 4301309805 00000100:00000010:0.0:1748873790.453290:0:6420:0:(niobuf.c:629:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 56 at ffff8800b6ce2bc0. 00000400:00000200:1.0:1748873790.453291:0:6417:0:(lib-move.c:5265:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000100:00000001:0.0:1748873790.453292:0:6420:0:(niobuf.c:641:kick_cpu_latency()) MGC192.168.202.156@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1748873790.453295:0:6420:0:(niobuf.c:1086:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1748873790.453296:0:5519:0:(import.c:825:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1748873790.453299:0:5519:0:(pinger.c:229:ptlrpc_pinger_process_import()) 106e5209-6aaf-48fa-ba19-a17064ac80d5->lustre-OST03e8_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1748873790.453299:0:6417:0:(lib-move.c:2934:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000100:00000001:0.0:1748873790.453301:0:6420:0:(client.c:1896:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1748873790.453302:0:5519:0:(pinger.c:247:ptlrpc_pinger_process_import()) 106e5209-6aaf-48fa-ba19-a17064ac80d5->lustre-OST03e8_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1748873790.453303:0:6417:0:(lib-move.c:2564:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1748873790.453304:0:5519:0:(pinger.c:229:ptlrpc_pinger_process_import()) 106e5209-6aaf-48fa-ba19-a17064ac80d5->lustre-OST2710_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1748873790.453304:0:6420:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1748873790.453306:0:5519:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST2710_UUID: starting recovery 00000400:00000200:1.0:1748873790.453306:0:6417:0:(lib-move.c:1750:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000100:00000001:2.0:1748873790.453308:0:5519:0:(import.c:693:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1748873790.453308:0:6417:0:(lib-move.c:1329:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1748873790.453309:0:5519:0:(import.c:66:import_set_state_nolock()) ffff8801373af000 lustre-OST2710_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1748873790.453311:0:5519:0:(import.c:480:import_select_connection()) Process entered 00000400:00000200:1.0:1748873790.453311:0:6417:0:(lib-move.c:1398:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1748873790.453313:0:6417:0:(lib-move.c:1870:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [2051:2051:255:4294967295] 192.168.202.156@tcp LPNI seq info [2051:2051:7:4294967295] 00000100:00000001:0.0:1748873790.453318:0:6420:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1748873790.453320:0:6420:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.453322:0:6420:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1748873790.453324:0:5519:0:(import.c:505:import_select_connection()) lustre-OST2710-osc-ffff8800a82f5800: connect to NID 192.168.202.156@tcp last attempt 6616 00000400:00000200:1.0:1748873790.453324:0:6417:0:(lib-move.c:1985:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000100:00000001:0.0:1748873790.453324:0:6420:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.453328:0:6420:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:1.0:1748873790.453329:0:6417:0:(socklnd_cb.c:1000:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000100:00000001:0.0:1748873790.453330:0:6420:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000400:00000200:2.0:1748873790.453331:0:5519:0:(peer.c:1541:LNetPeerDiscovered()) Peer NID 192.168.202.156@tcp is discovered: rc = 1 00000800:00000010:1.0:1748873790.453332:0:6417:0:(socklnd_cb.c:42:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8800b6398a00. 00000100:00000001:0.0:1748873790.453332:0:6420:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1748873790.453333:0:6420:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1748873790.453334:0:5519:0:(import.c:584:import_select_connection()) lustre-OST2710-osc-ffff8800a82f5800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1748873790.453334:0:6420:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1748873790.453337:0:5519:0:(lustre_net.h:1992:ptlrpc_connection_put()) PUT conn=ffff88012d03aa20 refcount 14 to 192.168.202.156@tcp 00000800:00000200:1.0:1748873790.453338:0:6417:0:(socklnd.c:225:ksocknal_find_peer_locked()) got peer_ni [ffff8800b3e4fd80] -> 12345-192.168.202.156@tcp (5) 00000100:00000001:2.0:1748873790.453339:0:5519:0:(connection.c:159:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1748873790.453341:0:5519:0:(connection.c:164:ptlrpc_connection_addref()) conn=ffff88012d03aa20 refcount 15 to 192.168.202.156@tcp 00000100:00000001:2.0:1748873790.453343:0:5519:0:(connection.c:166:ptlrpc_connection_addref()) Process leaving (rc=18446612137364400672 : -131936345150944 : ffff88012d03aa20) 00000020:00000001:2.0:1748873790.453344:0:5519:0:(genops.c:811:class_conn2export()) Process entered 00000800:00000200:1.0:1748873790.453344:0:6417:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.156@tcp ip 192.168.202.156:988 00000020:00000040:2.0:1748873790.453345:0:5519:0:(genops.c:823:class_conn2export()) looking for export cookie 0x696e714cbd1dbcb6 00000020:00000001:2.0:1748873790.453347:0:5519:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:1.0:1748873790.453348:0:6417:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8800b6398a00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1748873790.453351:0:6417:0:(niobuf.c:98:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1748873790.453353:0:5519:0:(lustre_handles.c:131:class_handle2object()) GET export ffff88013681d000 refcount=4 00000020:00000001:2.0:1748873790.453354:0:5519:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446612137523662848 : -131936185888768 : ffff88013681d000) 00000100:00000040:1.0:1748873790.453355:0:6417:0:(niobuf.c:600:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: using a timeout of 4802 usecs (5 jiffies) 00000020:00000001:2.0:1748873790.453356:0:5519:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446612137523662848 : -131936185888768 : ffff88013681d000) 00000100:00000001:1.0:1748873790.453358:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 0 is 4301308558 00000100:00000040:2.0:1748873790.453360:0:5519:0:(lustre_net.h:1992:ptlrpc_connection_put()) PUT conn=ffff88012d03aa20 refcount 14 to 192.168.202.156@tcp 00000100:00000001:1.0:1748873790.453360:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS request active for cpu 0, keep current deadline 4301309805 00000100:00000001:2.0:1748873790.453362:0:5519:0:(connection.c:159:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1748873790.453363:0:5519:0:(connection.c:164:ptlrpc_connection_addref()) conn=ffff88012d03aa20 refcount 15 to 192.168.202.156@tcp 00000100:00000001:1.0:1748873790.453363:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 1 is 4301308558 00000100:00000001:2.0:1748873790.453365:0:5519:0:(connection.c:166:ptlrpc_connection_addref()) Process leaving (rc=18446612137364400672 : -131936345150944 : ffff88012d03aa20) 00000100:00000001:1.0:1748873790.453365:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS request active for cpu 1, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453366:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 2 is 4301308558 00000020:00000040:2.0:1748873790.453367:0:5519:0:(genops.c:898:class_export_put()) PUTting export ffff88013681d000 : new refcount 3 00000100:00000001:1.0:1748873790.453368:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS request active for cpu 2, keep current deadline 4301309805 00000100:00080000:2.0:1748873790.453369:0:5519:0:(import.c:619:import_select_connection()) lustre-OST2710-osc-ffff8800a82f5800: import ffff8801373af000 using connection 192.168.202.156@tcp/192.168.202.156@tcp 00000100:00000001:1.0:1748873790.453369:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 3 is 4301308558 00000100:00000001:2.0:1748873790.453371:0:5519:0:(import.c:623:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453371:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8800a82f5800: PM QoS request active for cpu 3, keep current deadline 4301309805 00000100:00000001:0.0:1748873790.453372:0:6420:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1748873790.453373:0:5519:0:(sec.c:1538:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1748873790.453373:0:6417:0:(niobuf.c:1086:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1748873790.453373:0:6420:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1748873790.453374:0:6420:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1748873790.453375:0:5519:0:(sec.c:1578:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1748873790.453375:0:6417:0:(client.c:1896:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.453376:0:6420:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1748873790.453377:0:5519:0:(sec.c:1606:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453377:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.453379:0:5519:0:(obd_class.h:910:obd_reconnect()) Process entered 00000100:00000001:1.0:1748873790.453379:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1748873790.453380:0:5519:0:(osc_request.c:3772:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640078 ocd_version: 34617127 ocd_grant: 28672, lost: 0. 00000008:00000001:2.0:1748873790.453382:0:5519:0:(osc_request.c:3775:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.453384:0:5519:0:(obd_class.h:924:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453384:0:6417:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:2.0:1748873790.453386:0:5519:0:(client.c:585:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8801369c8700. 00000020:00000040:2.0:1748873790.453388:0:5519:0:(genops.c:1229:class_import_get()) import ffff8801373af000 refcount=6 obd=lustre-OST2710-osc-ffff8800a82f5800 00000100:00000001:1.0:1748873790.453389:0:6417:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1748873790.453390:0:5519:0:(sec.c:1984:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1748873790.453391:0:5519:0:(sec.c:1994:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453391:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1748873790.453392:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1748873790.453393:0:5519:0:(client.c:837:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1748873790.453394:0:5519:0:(sec.c:430:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1748873790.453394:0:6417:0:(client.c:1768:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1748873790.453395:0:5519:0:(sec.c:454:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1748873790.453398:0:5519:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8800b6cea000. 00000100:00000040:1.0:1748873790.453398:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1748873790.453401:0:5519:0:(client.c:917:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1748873790.453405:0:5519:0:(import.c:818:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8801369c8700 x1833823818221312/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1748873790.453406:0:6417:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1748873790.453407:0:6417:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1748873790.453409:0:6417:0:(sec.c:697:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1748873790.453411:0:6417:0:(sec.c:728:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1748873790.453412:0:5519:0:(jobid.c:925:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1748873790.453413:0:6417:0:(sec.c:756:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1748873790.453414:0:5519:0:(jobid.c:936:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1748873790.453414:0:6412:0:(socklnd_cb.c:540:ksocknal_process_transmit()) send(0) 0 00000100:00100000:1.0:1748873790.453415:0:6417:0:(client.c:1874:ptlrpc_send_new_req()) Sending RPC req@ffff8801369c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:106e5209-6aaf-48fa-ba19-a17064ac80d5:6417:1833823818221184:192.168.202.156@tcp:38: 00000100:00000040:2.0:1748873790.453417:0:5519:0:(ptlrpcd.c:236:ptlrpcd_add_req()) @@@ add req [ffff8801369c8700] to pc [ptlrpcd_rcv+-1] req@ffff8801369c8700 x1833823818221312/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1748873790.453417:0:6412:0:(socklnd_cb.c:99:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8800b6f3a200. 00000100:00000001:1.0:1748873790.453418:0:6417:0:(niobuf.c:817:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1748873790.453420:0:6417:0:(sec.c:1047:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:0.0:1748873790.453421:0:6412:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:1.0:1748873790.453422:0:6417:0:(sec.c:1084:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1748873790.453423:0:6417:0:(sec.c:1796:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1748873790.453425:0:6417:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8800ab5c2800. 02000000:00000001:1.0:1748873790.453426:0:6417:0:(sec.c:1806:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1748873790.453426:0:6412:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000400:00000010:1.0:1748873790.453429:0:6417:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8800aa0144d0. 00000400:00000200:0.0:1748873790.453430:0:6412:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800aa3b92a8 00000400:00000010:0.0:1748873790.453432:0:6412:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800aa3b92a8. 00000400:00000010:1.0:1748873790.453435:0:6417:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800ab5d3198. 00000100:00000001:0.0:1748873790.453435:0:6412:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1748873790.453436:0:5519:0:(import.c:825:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1748873790.453438:0:6417:0:(niobuf.c:1048:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1833823818221184, portal 10 00000100:00000001:1.0:1748873790.453439:0:6417:0:(client.c:3219:ptlrpc_request_addref()) Process entered 00000100:00000200:0.0:1748873790.453439:0:6412:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.453440:0:6417:0:(client.c:3221:ptlrpc_request_addref()) Process leaving (rc=18446612137525411840 : -131936184139776 : ffff8801369c8000) 00000100:00000040:2.0:1748873790.453441:0:5519:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (6646) 00000100:00000040:1.0:1748873790.453446:0:6417:0:(niobuf.c:1068:ptl_send_rpc()) @@@ send flags=200 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.453448:0:6412:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1748873790.453451:0:6412:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.453452:0:6417:0:(niobuf.c:50:ptl_send_buf()) Process entered 00000100:00000040:1.0:1748873790.453455:0:6417:0:(niobuf.c:53:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000400:00000010:1.0:1748873790.453456:0:6417:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800ab5d3440. 00000100:00000001:0.0:1748873790.453457:0:6412:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1748873790.453458:0:6417:0:(niobuf.c:81:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1833823818221184, offset 0 00000100:00000001:0.0:1748873790.453459:0:6412:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000200:1.0:1748873790.453462:0:6417:0:(lib-move.c:5265:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1748873790.453467:0:6417:0:(lib-move.c:2934:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1748873790.453470:0:6417:0:(lib-move.c:2564:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1748873790.453472:0:6417:0:(lib-move.c:1750:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1748873790.453479:0:6417:0:(lib-move.c:1329:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1748873790.453481:0:6417:0:(lib-move.c:1398:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1748873790.453484:0:6417:0:(lib-move.c:1870:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [2052:2052:255:4294967295] 192.168.202.156@tcp LPNI seq info [2052:2052:7:4294967295] 00000400:00000200:1.0:1748873790.453492:0:6417:0:(lib-move.c:1985:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1748873790.453496:0:6417:0:(socklnd_cb.c:1000:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1748873790.453499:0:6417:0:(socklnd_cb.c:42:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8800b6398d00. 00000800:00000200:1.0:1748873790.453502:0:6417:0:(socklnd.c:225:ksocknal_find_peer_locked()) got peer_ni [ffff8800b3e4fd80] -> 12345-192.168.202.156@tcp (5) 00000800:00000200:1.0:1748873790.453507:0:6417:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.156@tcp ip 192.168.202.156:988 00000800:00000200:0.0:1748873790.453508:0:6412:0:(socklnd_cb.c:540:ksocknal_process_transmit()) send(0) 0 00000800:00000200:1.0:1748873790.453509:0:6417:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8800b6398d00 type 1, nob 616 niov 1 nkiov 1 00000800:00000010:0.0:1748873790.453511:0:6412:0:(socklnd_cb.c:99:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8800b6398a00. 00000100:00000001:1.0:1748873790.453512:0:6417:0:(niobuf.c:98:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1748873790.453514:0:6417:0:(niobuf.c:600:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: using a timeout of 2596 usecs (3 jiffies) 00000400:00000200:0.0:1748873790.453514:0:6412:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1748873790.453517:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 0 is 4301308556 00000100:00000001:1.0:1748873790.453518:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS request active for cpu 0, keep current deadline 4301309805 00000400:00000200:0.0:1748873790.453518:0:6412:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000100:00000001:1.0:1748873790.453520:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 1 is 4301308556 00000400:00000200:0.0:1748873790.453520:0:6412:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800ab5d3110 00000400:00000010:0.0:1748873790.453521:0:6412:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800ab5d3110. 00000100:00000001:1.0:1748873790.453522:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS request active for cpu 1, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453524:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 2 is 4301308556 00000100:00000001:0.0:1748873790.453524:0:6412:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1748873790.453525:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS request active for cpu 2, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453527:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 3 is 4301308556 00000100:00000001:1.0:1748873790.453528:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8800a82f5800: PM QoS request active for cpu 3, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453530:0:6417:0:(niobuf.c:1086:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1748873790.453532:0:6417:0:(client.c:1896:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1748873790.453533:0:6412:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.453534:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453535:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.453539:0:6417:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1748873790.453540:0:6417:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1748873790.453541:0:6412:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1748873790.453542:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1748873790.453543:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1748873790.453543:0:6412:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.453544:0:6417:0:(client.c:1768:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1748873790.453547:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8801369c8700 x1833823818221312/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.453549:0:6412:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.453550:0:6412:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1748873790.453559:0:6417:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1748873790.453560:0:6417:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1748873790.453562:0:6417:0:(sec.c:697:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1748873790.453563:0:6417:0:(sec.c:728:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1748873790.453565:0:6417:0:(sec.c:756:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1748873790.453568:0:6417:0:(client.c:1874:ptlrpc_send_new_req()) Sending RPC req@ffff8801369c8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:106e5209-6aaf-48fa-ba19-a17064ac80d5:6417:1833823818221312:192.168.202.156@tcp:8: 00000100:00000001:1.0:1748873790.453571:0:6417:0:(niobuf.c:817:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1748873790.453573:0:6417:0:(sec.c:1047:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1748873790.453574:0:6417:0:(sec.c:1084:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1748873790.453575:0:6417:0:(sec.c:1796:sptlrpc_cli_alloc_repbuf()) Process entered 00000800:00000200:0.0:1748873790.453575:0:6412:0:(socklnd_cb.c:540:ksocknal_process_transmit()) send(0) 0 02000000:00000010:1.0:1748873790.453577:0:6417:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8800ab5c2000. 00000800:00000010:0.0:1748873790.453577:0:6412:0:(socklnd_cb.c:99:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8800b6398d00. 02000000:00000001:1.0:1748873790.453579:0:6417:0:(sec.c:1806:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1748873790.453579:0:6412:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1748873790.453582:0:6417:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff8800aa014b58. 00000400:00000200:0.0:1748873790.453583:0:6412:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000400:00000010:1.0:1748873790.453584:0:6417:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800ab5d3bb0. 00000400:00000200:0.0:1748873790.453585:0:6412:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800ab5d3440 00000400:00000010:0.0:1748873790.453587:0:6412:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800ab5d3440. 00000100:00000001:0.0:1748873790.453589:0:6412:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:1.0:1748873790.453590:0:6417:0:(niobuf.c:1048:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1833823818221312, portal 4 00000100:00000200:0.0:1748873790.453591:0:6412:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.453592:0:6417:0:(client.c:3219:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1748873790.453593:0:6417:0:(client.c:3221:ptlrpc_request_addref()) Process leaving (rc=18446612137525413632 : -131936184137984 : ffff8801369c8700) 00000100:00000040:1.0:1748873790.453598:0:6417:0:(niobuf.c:1068:ptl_send_rpc()) @@@ send flags=200 req@ffff8801369c8700 x1833823818221312/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1748873815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.453598:0:6412:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1748873790.453601:0:6412:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.453604:0:6417:0:(niobuf.c:50:ptl_send_buf()) Process entered 00000100:00000040:1.0:1748873790.453607:0:6417:0:(niobuf.c:53:ptl_send_buf()) peer_id 12345-192.168.202.156@tcp 00000100:00000001:0.0:1748873790.453607:0:6412:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1748873790.453608:0:6417:0:(lib-md.c:130:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8800ab5d3d48. 00000100:00000001:0.0:1748873790.453608:0:6412:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000200:1.0:1748873790.453610:0:6417:0:(niobuf.c:81:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1833823818221312, offset 0 00000400:00000200:1.0:1748873790.453613:0:6417:0:(lib-move.c:5265:LNetPut()) LNetPut -> 12345-192.168.202.156@tcp 00000400:00000200:1.0:1748873790.453618:0:6417:0:(lib-move.c:2934:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.202.156@tcp local destination 00000400:00000200:1.0:1748873790.453620:0:6417:0:(lib-move.c:2564:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1748873790.453622:0:6417:0:(lib-move.c:1750:lnet_get_best_ni()) selected best_ni 192.168.202.56@tcp 00000400:00000200:1.0:1748873790.453624:0:6417:0:(lib-move.c:1329:lnet_select_peer_ni()) 192.168.202.56@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1748873790.453626:0:6417:0:(lib-move.c:1398:lnet_select_peer_ni()) sd_best_lpni = 192.168.202.156@tcp 00000400:00000200:1.0:1748873790.453629:0:6417:0:(lib-move.c:1870:lnet_handle_send()) 192.168.202.56@tcp NI seq info: [2053:2053:256:4294967295] 192.168.202.156@tcp LPNI seq info [2053:2053:8:4294967295] 00000400:00000200:1.0:1748873790.453666:0:6417:0:(lib-move.c:1985:lnet_handle_send()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp:) -> 192.168.202.156@tcp(192.168.202.156@tcp:192.168.202.156@tcp) : PUT try# 0 00000800:00000200:1.0:1748873790.453676:0:6417:0:(socklnd_cb.c:1000:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.202.156@tcp 00000800:00000010:1.0:1748873790.453679:0:6417:0:(socklnd_cb.c:42:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8800b6398000. 00000800:00000200:1.0:1748873790.453682:0:6417:0:(socklnd.c:225:ksocknal_find_peer_locked()) got peer_ni [ffff8800b3e4fd80] -> 12345-192.168.202.156@tcp (5) 00000800:00000200:1.0:1748873790.453687:0:6417:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.156@tcp ip 192.168.202.156:988 00000800:00000200:1.0:1748873790.453689:0:6417:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff8800b6398000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1748873790.453711:0:6417:0:(niobuf.c:98:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1748873790.453714:0:6417:0:(niobuf.c:600:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: using a timeout of 2932 usecs (3 jiffies) 00000100:00000001:1.0:1748873790.453716:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 0 is 4301308557 00000100:00000001:1.0:1748873790.453718:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS request active for cpu 0, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453720:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 1 is 4301308557 00000100:00000001:1.0:1748873790.453722:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS request active for cpu 1, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453723:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 2 is 4301308557 00000100:00000001:1.0:1748873790.453725:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS request active for cpu 2, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453726:0:6417:0:(niobuf.c:626:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS new deadline estimation for cpu 3 is 4301308557 00000100:00000001:1.0:1748873790.453728:0:6417:0:(niobuf.c:655:kick_cpu_latency()) lustre-OST2710-osc-ffff8800a82f5800: PM QoS request active for cpu 3, keep current deadline 4301309805 00000100:00000001:1.0:1748873790.453729:0:6417:0:(niobuf.c:1086:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1748873790.453731:0:6417:0:(client.c:1896:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453734:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453736:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.453739:0:6417:0:(client.c:2587:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1748873790.453740:0:6417:0:(client.c:2615:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1748873790.453742:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1748873790.453743:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1748873790.453744:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.453745:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1748873790.453805:0:6411:0:(socklnd_cb.c:540:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1748873790.453808:0:6411:0:(socklnd_cb.c:99:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8800b6398000. 00000400:00000200:0.0:1748873790.453811:0:6411:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1748873790.453815:0:6411:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000400:00000200:0.0:1748873790.453818:0:6411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800ab5d3d48 00000400:00000010:0.0:1748873790.453819:0:6411:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800ab5d3d48. 00000100:00000001:0.0:1748873790.453821:0:6411:0:(events.c:35:request_out_callback()) Process entered 00000100:00000200:0.0:1748873790.453825:0:6411:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8801369c8700 x1833823818221312/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1748873815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.453833:0:6411:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1748873790.453835:0:6411:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8801369c8700 x1833823818221312/t0(0) o8->lustre-OST2710-osc-ffff8800a82f5800@192.168.202.156@tcp:28/4 lens 520/544 e 0 to 0 dl 1748873815 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1748873790.453854:0:6411:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1748873790.453855:0:6411:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000001:2.0:1748873790.455425:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455436:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1748873790.455439:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455441:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1748873790.455449:0:6413:0:(lib-move.c:4755:lnet_parse()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp) <- 192.168.202.156@tcp : PUT - for me 00000400:00000200:2.0:1748873790.455459:0:6413:0:(lib-ptl.c:583:lnet_ptl_match_md()) Request from 12345-192.168.202.156@tcp of length 224 into portal 25 MB=0x683da61440580 00000400:00000200:2.0:1748873790.455465:0:6413:0:(lib-ptl.c:209:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.202.156@tcp of length 224/224 into md 0x40f9 [1] + 224 00000800:00000001:2.0:1748873790.455470:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455473:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1748873790.455474:0:6413:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1748873790.455477:0:6413:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000100:00000001:2.0:1748873790.455480:0:6413:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000200:2.0:1748873790.455484:0:6413:0:(events.c:81:reply_in_callback()) @@@ type 2, status 0 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1748873790.455493:0:6413:0:(events.c:150:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1748873790.455516:0:6413:0:(events.c:164:reply_in_callback()) Process leaving 00000800:00000001:2.0:1748873790.455522:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455524:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1748873790.455526:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455527:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1748873790.455530:0:6413:0:(lib-move.c:4755:lnet_parse()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp) <- 192.168.202.156@tcp : PUT - for me 00000400:00000200:2.0:1748873790.455534:0:6413:0:(lib-ptl.c:583:lnet_ptl_match_md()) Request from 12345-192.168.202.156@tcp of length 416 into portal 10 MB=0x683da61440600 00000400:00000200:2.0:1748873790.455537:0:6413:0:(lib-ptl.c:209:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.156@tcp of length 416/416 into md 0x4101 [1] + 0 00000800:00000001:2.0:1748873790.455540:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455542:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1748873790.455543:0:6413:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1748873790.455546:0:6413:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000100:00000001:2.0:1748873790.455555:0:6413:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000200:2.0:1748873790.455557:0:6413:0:(events.c:81:reply_in_callback()) @@@ type 2, status 0 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1748873790.455565:0:6413:0:(events.c:150:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1748873790.455573:0:6413:0:(events.c:164:reply_in_callback()) Process leaving 00000800:00000001:2.0:1748873790.455576:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455578:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1748873790.455580:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455581:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1748873790.455584:0:6413:0:(lib-move.c:4755:lnet_parse()) TRACE: 192.168.202.56@tcp(192.168.202.56@tcp) <- 192.168.202.156@tcp : PUT - for me 00000400:00000200:2.0:1748873790.455588:0:6413:0:(lib-ptl.c:583:lnet_ptl_match_md()) Request from 12345-192.168.202.156@tcp of length 416 into portal 10 MB=0x683da61440680 00000100:00000001:1.0:1748873790.455588:0:6420:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1748873790.455590:0:6413:0:(lib-ptl.c:209:lnet_try_match_md()) Incoming put index a from 12345-192.168.202.156@tcp of length 416/416 into md 0x4109 [1] + 0 00000100:00000001:1.0:1748873790.455592:0:6420:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000800:00000001:2.0:1748873790.455593:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000400:00000001:1.0:1748873790.455596:0:6420:0:(lib-msg.c:21:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1748873790.455597:0:6420:0:(lib-msg.c:30:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1748873790.455599:0:6420:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8800b6e440b0. 00000800:00000001:2.0:1748873790.455600:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1748873790.455602:0:6413:0:(lib-msg.c:1049:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1748873790.455603:0:6420:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800aa3b9220 00000400:00000200:2.0:1748873790.455604:0:6413:0:(lib-msg.c:821:lnet_health_check()) health check: 192.168.202.56@tcp->192.168.202.156@tcp: PUT: OK 00000400:00000010:1.0:1748873790.455611:0:6420:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800aa3b9220. 00000100:00000001:1.0:1748873790.455614:0:6420:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000001:2.0:1748873790.455615:0:6413:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000200:2.0:1748873790.455618:0:6413:0:(events.c:81:reply_in_callback()) @@@ type 2, status 0 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1748873790.455619:0:6420:0:(events.c:81:reply_in_callback()) @@@ type 6, status 0 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1748873790.455625:0:6413:0:(events.c:150:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1748873790.455628:0:6420:0:(events.c:103:reply_in_callback()) @@@ unlink req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1748873790.455631:0:6413:0:(events.c:164:reply_in_callback()) Process leaving 00000800:00000001:2.0:1748873790.455633:0:6413:0:(socklnd_cb.c:346:ksocknal_receive()) Process entered 00000800:00000001:2.0:1748873790.455664:0:6413:0:(socklnd_cb.c:384:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.455667:0:6420:0:(events.c:164:reply_in_callback()) Process leaving 00000100:00000001:1.0:1748873790.455669:0:6420:0:(client.c:2932:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.455671:0:6420:0:(client.c:1550:after_reply()) Process entered 02000000:00000001:1.0:1748873790.455673:0:6420:0:(sec.c:1092:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1748873790.455675:0:6420:0:(pack_generic.c:603:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1748873790.455677:0:6420:0:(pack_generic.c:621:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1748873790.455680:0:6420:0:(sec.c:1148:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1748873790.455684:0:6420:0:(import.c:2097:obd_at_measure()) add 5 to ffff8800b63a2c38 time=77 v=5 (5 5 5 5) 00000100:00001000:1.0:1748873790.455687:0:6420:0:(import.c:2097:obd_at_measure()) add 1 to ffff8800b63a2bf0 time=15 v=5 (1 1 1 1) 00000100:00000001:1.0:1748873790.455689:0:6420:0:(client.c:1472:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1748873790.455691:0:6420:0:(client.c:1490:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1748873790.455693:0:6420:0:(ldlm_request.c:1515:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1748873790.455694:0:6420:0:(ldlm_request.c:1519:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.455697:0:6420:0:(client.c:1752:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1748873790.455700:0:6420:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.455706:0:6420:0:(client.c:2300:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1748873790.455708:0:6420:0:(niobuf.c:441:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1748873790.455709:0:6420:0:(niobuf.c:455:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1748873790.455712:0:6420:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1748873790.455718:0:6420:0:(client.c:2365:ptlrpc_check_set()) Completed RPC req@ffff8801369ca300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_02:05a09d41-c85d-4993-a6db-87da69cb8def:6420:1833823818220928:192.168.202.156@tcp:400: 00000100:00000001:1.0:1748873790.455722:0:6420:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.455723:0:6420:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1748873790.455724:0:6420:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.455726:0:6420:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.455727:0:6420:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1748873790.455734:0:6420:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8801369ca300 x1833823818220928/t0(0) o400->MGC192.168.202.156@tcp@192.168.202.156@tcp:26/25 lens 224/224 e 0 to 0 dl 1748873806 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1748873790.455739:0:6420:0:(client.c:2741:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1748873790.455741:0:6420:0:(sec.c:1818:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1748873790.455742:0:6420:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8800b3714c00. 02000000:00000001:1.0:1748873790.455745:0:6420:0:(sec.c:1832:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1748873790.455746:0:6420:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1748873790.455747:0:6420:0:(genops.c:1242:class_import_put()) import ffff8800b63a2800 refcount=3 obd=MGC192.168.202.156@tcp 00000020:00000001:1.0:1748873790.455748:0:6420:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1748873790.455749:0:6420:0:(sec_null.c:173:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 256 at ffff8800b6dcf900. 02000000:00000001:1.0:1748873790.455751:0:6420:0:(sec.c:468:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1748873790.455752:0:6420:0:(sec.c:485:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1748873790.455753:0:6420:0:(client.c:591:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8801369ca300. 00000100:00000001:3.0:1748873790.455754:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1748873790.455755:0:6420:0:(client.c:2799:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1748873790.455756:0:6420:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.455758:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1748873790.455758:0:6420:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1748873790.455760:0:6420:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1748873790.455761:0:6420:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1748873790.455762:0:6417:0:(lib-msg.c:21:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1748873790.455762:0:6420:0:(client.c:2924:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.455763:0:6420:0:(client.c:1356:ptlrpc_import_delay_req()) Process entered 00000400:00000001:3.0:1748873790.455764:0:6417:0:(lib-msg.c:30:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1748873790.455764:0:6420:0:(client.c:1414:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1748873790.455765:0:6420:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1748873790.455766:0:6417:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8800aa0144d0. 00000100:00000001:1.0:1748873790.455766:0:6420:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1748873790.455770:0:6417:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800ab5d3198 00000400:00000010:3.0:1748873790.455772:0:6417:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800ab5d3198. 00000100:00000001:3.0:1748873790.455775:0:6417:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000200:3.0:1748873790.455801:0:6417:0:(events.c:81:reply_in_callback()) @@@ type 6, status 0 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1748873790.455811:0:6417:0:(events.c:103:reply_in_callback()) @@@ unlink req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1748873790.455818:0:6417:0:(events.c:164:reply_in_callback()) Process leaving 00000100:00000001:3.0:1748873790.455819:0:6417:0:(client.c:2932:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.455821:0:6417:0:(client.c:1550:after_reply()) Process entered 02000000:00000001:3.0:1748873790.455823:0:6417:0:(sec.c:1092:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1748873790.455824:0:6417:0:(pack_generic.c:603:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1748873790.455826:0:6417:0:(pack_generic.c:621:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1748873790.455827:0:6417:0:(sec.c:1148:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1748873790.455832:0:6417:0:(import.c:2097:obd_at_measure()) add 0 to ffff8801373abc38 time=3077 v=5 (5 0 0 0) 00000100:00001000:3.0:1748873790.455835:0:6417:0:(import.c:2097:obd_at_measure()) add 1 to ffff8801373abbf0 time=77 v=5 (1 1 1 1) 00000100:00000001:3.0:1748873790.455837:0:6417:0:(client.c:1472:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1748873790.455838:0:6417:0:(client.c:1484:ptlrpc_check_status()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000100:00000001:3.0:1748873790.455841:0:6417:0:(client.c:3040:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1748873790.455843:0:6417:0:(client.c:3052:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff8800a82f5800: skip recheck: last_committed 0 00000100:00000001:3.0:1748873790.455845:0:6417:0:(client.c:3053:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1748873790.455846:0:6417:0:(client.c:1752:after_reply()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000100:00000040:3.0:1748873790.455849:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/416 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQU/200/0 rc -11/-11 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1748873790.455855:0:6417:0:(client.c:2300:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1748873790.455857:0:6417:0:(niobuf.c:441:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1748873790.455858:0:6417:0:(niobuf.c:455:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.455867:0:6417:0:(import.c:1037:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1748873790.455868:0:6417:0:(import.c:1068:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00080000:3.0:1748873790.455870:0:6417:0:(import.c:66:import_set_state_nolock()) ffff8801373ab800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1748873790.455872:0:6417:0:(import.c:1483:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.156@tcp failed (-11) 00000100:00000001:3.0:1748873790.455874:0:6417:0:(import.c:1489:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000100:00000040:3.0:1748873790.455877:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/416 e 0 to 0 dl 1748873800 ref 1 fl Interpret:RNQU/200/0 rc -11/-11 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1748873790.455883:0:6417:0:(client.c:2365:ptlrpc_check_set()) Completed RPC req@ffff8801369c8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:106e5209-6aaf-48fa-ba19-a17064ac80d5:6417:1833823818221184:192.168.202.156@tcp:38: 00000400:00000001:3.0:1748873790.455888:0:6417:0:(lib-msg.c:21:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1748873790.455889:0:6417:0:(lib-msg.c:30:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1748873790.455890:0:6417:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff8800aa014580. 00000400:00000200:3.0:1748873790.455892:0:6417:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8800ab5d3f68 00000400:00000010:3.0:1748873790.455893:0:6417:0:(lib-lnet.h:298:lnet_md_free()) slab-freed 'md': 136 at ffff8800ab5d3f68. 00000100:00000001:3.0:1748873790.455895:0:6417:0:(events.c:79:reply_in_callback()) Process entered 00000100:00000200:3.0:1748873790.455897:0:6417:0:(events.c:81:reply_in_callback()) @@@ type 6, status 0 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1748873790.455903:0:6417:0:(events.c:103:reply_in_callback()) @@@ unlink req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/544 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1748873790.455908:0:6417:0:(events.c:164:reply_in_callback()) Process leaving 00000100:00000001:3.0:1748873790.455909:0:6417:0:(client.c:2932:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.455911:0:6417:0:(client.c:1550:after_reply()) Process entered 02000000:00000001:3.0:1748873790.455912:0:6417:0:(sec.c:1092:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1748873790.455913:0:6417:0:(pack_generic.c:603:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1748873790.455914:0:6417:0:(pack_generic.c:621:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1748873790.455915:0:6417:0:(sec.c:1148:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1748873790.455918:0:6417:0:(import.c:2097:obd_at_measure()) add 0 to ffff8800aa0cc438 time=3076 v=5 (5 0 0 0) 00000100:00001000:3.0:1748873790.455920:0:6417:0:(import.c:2097:obd_at_measure()) add 1 to ffff8800aa0cc3f0 time=76 v=5 (1 1 1 1) 00000100:00000001:3.0:1748873790.455922:0:6417:0:(client.c:1472:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1748873790.455922:0:6417:0:(client.c:1484:ptlrpc_check_status()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000100:00000001:3.0:1748873790.455924:0:6417:0:(client.c:3040:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1748873790.455925:0:6417:0:(client.c:3052:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8800a82f5800: skip recheck: last_committed 4294967307 00000100:00000001:3.0:1748873790.455929:0:6417:0:(client.c:3053:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1748873790.455931:0:6417:0:(client.c:1752:after_reply()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000100:00000040:3.0:1748873790.455934:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/416 e 0 to 0 dl 1748873800 ref 1 fl Rpc:RNQU/200/0 rc -11/-11 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1748873790.455939:0:6417:0:(client.c:2300:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1748873790.455941:0:6417:0:(niobuf.c:441:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1748873790.455941:0:6417:0:(niobuf.c:455:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.455943:0:6417:0:(import.c:1037:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1748873790.455943:0:6417:0:(import.c:1068:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00080000:3.0:1748873790.455945:0:6417:0:(import.c:66:import_set_state_nolock()) ffff8800aa0cc000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1748873790.455947:0:6417:0:(import.c:1483:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.156@tcp failed (-11) 00000100:00000001:3.0:1748873790.455948:0:6417:0:(import.c:1489:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000100:00000040:3.0:1748873790.455950:0:6417:0:(lustre_net.h:2472:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/416 e 0 to 0 dl 1748873800 ref 1 fl Interpret:RNQU/200/0 rc -11/-11 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1748873790.455956:0:6417:0:(client.c:2365:ptlrpc_check_set()) Completed RPC req@ffff8801369cb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:106e5209-6aaf-48fa-ba19-a17064ac80d5:6417:1833823818221056:192.168.202.156@tcp:38: 00000100:00000001:3.0:1748873790.455959:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1748873790.455961:0:6417:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1748873790.455963:0:6417:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8801369c8000 x1833823818221184/t0(0) o38->lustre-MDT0001-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/416 e 0 to 0 dl 1748873800 ref 1 fl Complete:RNQU/200/0 rc -11/-11 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1748873790.455968:0:6417:0:(client.c:2741:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1748873790.455969:0:6417:0:(sec.c:1818:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1748873790.455971:0:6417:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8800ab5c2800. 02000000:00000001:3.0:1748873790.455973:0:6417:0:(sec.c:1832:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1748873790.455974:0:6417:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1748873790.455975:0:6417:0:(genops.c:1242:class_import_put()) import ffff8801373ab800 refcount=5 obd=lustre-MDT0001-mdc-ffff8800a82f5800 00000020:00000001:3.0:1748873790.455977:0:6417:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1748873790.455978:0:6417:0:(sec_null.c:173:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at ffff8800b6ce9800. 02000000:00000001:3.0:1748873790.455980:0:6417:0:(sec.c:468:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1748873790.455981:0:6417:0:(sec.c:485:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1748873790.455982:0:6417:0:(client.c:591:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8801369c8000. 00000100:00000001:3.0:1748873790.455987:0:6417:0:(client.c:2799:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1748873790.455988:0:6417:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.455989:0:6417:0:(client.c:2817:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1748873790.455992:0:6417:0:(client.c:2825:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8801369cb100 x1833823818221056/t0(0) o38->lustre-MDT0000-mdc-ffff8800a82f5800@192.168.202.156@tcp:12/10 lens 520/416 e 0 to 0 dl 1748873800 ref 1 fl Complete:RNQU/200/0 rc -11/-11 job:'' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1748873790.455997:0:6417:0:(client.c:2741:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1748873790.455998:0:6417:0:(sec.c:1818:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1748873790.455999:0:6417:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8800ab5c1c00. 02000000:00000001:3.0:1748873790.456001:0:6417:0:(sec.c:1832:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1748873790.456002:0:6417:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1748873790.456002:0:6417:0:(genops.c:1242:class_import_put()) import ffff8800aa0cc000 refcount=10 obd=lustre-MDT0000-mdc-ffff8800a82f5800 00000020:00000001:3.0:1748873790.456004:0:6417:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1748873790.456004:0:6417:0:(sec_null.c:173:null_free_reqbuf()) kfreed 'req->rq_reqbuf': 1024 at ffff8800b6ceb400. 02000000:00000001:3.0:1748873790.456006:0:6417:0:(sec.c:468:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1748873790.456007:0:6417:0:(sec.c:485:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1748873790.456008:0:6417:0:(client.c:591:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8801369cb100. 00000100:00000001:3.0:1748873790.456009:0:6417:0:(client.c:2799:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1748873790.456010:0:6417:0:(client.c:2858:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1748873790.456011:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1748873790.456013:0:6417:0:(ptlrpcd.c:253:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1748873790.456014:0:6417:0:(client.c:1945:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1748873790.456015:0:6417:0:(client.c:2416:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1748873790.456016:0:6417:0:(ptlrpcd.c:352:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1748873790.709775:0:5799:0:(llite_lib.c:4119:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1748873790.809750:0:24853:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1748873790.809756:0:24853:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1748873790.809761:0:24853:0:(ldlm_pool.c:459:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1748873790.809762:0:24853:0:(ldlm_pool.c:463:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1748873790.809872:0:10383:0:(genops.c:1912:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1748873790.809877:0:10383:0:(genops.c:1926:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1748873790.809881:0:10383:0:(ldlm_lockd.c:2816:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:00000001:3.0:1748873790.820435:0:5842:0:(debug.c:791:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:3.0:1748873790.820439:0:5842:0:(debug.c:792:libcfs_debug_mark_buffer()) DEBUG MARKER: conf-sanity test_57b: @@@@@@ FAIL: import is not in FULL state 00000001:00000001:3.0:1748873790.824502:0:5842:0:(debug.c:794:libcfs_debug_mark_buffer()) ************************************************** 00000080:00000001:1.0:1748873791.240148:0:5951:0:(llite_lib.c:4119:ll_show_options()) Process leaving (rc=0 : 0 : 0) Debug log: 733 lines, 733 kept, 0 dropped, 0 bad.