00000100:00000040:3.0F:1787585190.869028:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585190.869048:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585190.869123:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0c700. 00000020:00000040:3.0:1787585190.869130:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f60d3c000 refcount=4 obd=MGC192.168.203.132@tcp 00000100:00000001:3.0:1787585190.869134:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585190.869137:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585190.869143:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585190.869180:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9300. 00000100:00000001:3.0:1787585190.869188:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869273:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585190.869285:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585190.869343:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869350:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a739ccc1] to pc [ptlrpcd_01_01+1] req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.869374:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869378:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585190.869381:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585190.869402:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0ea00. 00000020:00000040:3.0:1787585190.869405:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f482d8000 refcount=9 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000100:00000001:3.0:1787585190.869407:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585190.869409:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585190.869411:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585190.869414:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9900. 00000100:00000001:3.0:1787585190.869417:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869422:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585190.869428:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585190.869430:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869434:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009afa0799] to pc [ptlrpcd_01_00+0] req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.869442:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869450:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585190.869453:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585190.869456:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0f480. 00000020:00000040:3.0:1787585190.869459:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f482dc000 refcount=4 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000100:00000001:3.0:1787585190.869461:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585190.869462:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585190.869464:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585190.869468:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9100. 00000100:00000001:3.0:1787585190.869471:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869476:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585190.869483:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585190.869485:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869489:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005cec7c32] to pc [ptlrpcd_01_01+1] req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.869495:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869498:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585190.869500:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585190.869503:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0df80. 00000020:00000040:3.0:1787585190.869506:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f48b96800 refcount=4 obd=lustre-OST0000-osc-ffff940f48b94800 00000100:00000001:3.0:1787585190.869508:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585190.869509:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585190.869511:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585190.869514:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9e00. 00000100:00000001:3.0:1787585190.869517:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869521:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585190.869528:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585190.869530:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869533:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092ab15fa] to pc [ptlrpcd_01_00+0] req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.869551:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.869554:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787585190.869557:0:232188:0:(pinger.c:247:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787585190.869561:0:232188:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13012) 00000100:00000001:3.0:1787585190.869577:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.869581:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585190.869583:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787585190.869588:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.869596:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787585190.869598:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.869602:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787585190.869604:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787585190.869607:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787585190.869610:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2413:1874405500437888:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.869614:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787585190.869619:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787585190.869621:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.869624:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787585190.869628:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f4605ba00. 02000000:00000001:3.0:1787585190.869630:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585190.869637:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:3.0:1787585190.869641:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f434dedc0. 00000400:00000010:3.0:1787585190.869649:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd2a8. 00000100:00000200:3.0:1787585190.869656:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500437888, portal 25 00000100:00000001:3.0:1787585190.869658:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787585190.869660:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802449152 : -118680907102464 : ffff940f72b0c700) 00000100:00000040:3.0:1787585190.869666:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.869673:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787585190.869855:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:3.0:1787585190.869858:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd550. 00000100:00000200:3.0:1787585190.869862:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1874405500437888, offset 0 00000400:00000200:3.0:1787585190.869869:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:3.0:1787585190.869879:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.869886:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:3.0:1787585190.869891:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787585190.869896:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:3.0:1787585190.869900:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787585190.869903:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:3.0:1787585190.869907:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538716:538716:256:4294967295] 192.168.203.132@tcp LPNI seq info [538681:538681:8:4294967295] 00000400:00000200:3.0:1787585190.869912:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.869921:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:3.0:1787585190.869926:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:3.0:1787585190.869932:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940e410e9f00. 00000800:00000200:3.0:1787585190.869939:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:3.0:1787585190.869946:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585190.869950:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940e410e9f00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787585190.869962:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585190.869966:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:3.0:1787585190.869969:0:2413:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 2 is 4307678428 00000100:00000010:3.0:1787585190.869974:0:2413:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f714ca180. 00000100:00000001:3.0:1787585190.869980:0:2413:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request now active for cpu 2 00000100:00000001:3.0:1787585190.869985:0:2413:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 3 is 4307678428 00000100:00000010:3.0:1787585190.869987:0:2413:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f714cab40. 00000100:00000001:3.0:1787585190.870001:0:2413:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request now active for cpu 3 00000100:00000001:3.0:1787585190.870005:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.870008:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.870011:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787585190.870015:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.870033:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787585190.870035:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.870037:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787585190.870039:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787585190.870041:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787585190.870045:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438144:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.870048:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787585190.870130:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787585190.870133:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.870134:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787585190.870139:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f4605b400. 02000000:00000001:3.0:1787585190.870141:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585190.870145:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:3.0:1787585190.870148:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f434de840. 00000400:00000010:3.0:1787585190.870152:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd770. 00000100:00000200:3.0:1787585190.870155:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438144, portal 10 00000100:00000001:3.0:1787585190.870157:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787585190.870158:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802460800 : -118680907090816 : ffff940f72b0f480) 00000100:00000040:3.0:1787585190.870166:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.870173:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787585190.870175:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:3.0:1787585190.870178:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604ddd48. 00000100:00000200:3.0:1787585190.870180:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500438144, offset 0 00000400:00000200:3.0:1787585190.870185:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:3.0:1787585190.870189:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.870265:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:3.0:1787585190.870269:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787585190.870272:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:3.0:1787585190.870274:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787585190.870277:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:3.0:1787585190.870287:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538717:538717:255:4294967295] 192.168.203.132@tcp LPNI seq info [538682:538682:7:4294967295] 00000400:00000200:3.0:1787585190.870361:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.870368:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:3.0:1787585190.870374:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:3.0:1787585190.870378:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940e410e9a00. 00000800:00000200:3.0:1787585190.870410:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:3.0:1787585190.870416:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585190.870419:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940e410e9a00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787585190.870422:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.870426:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: using a timeout of 25401 usecs (26 jiffies) 00000400:00000200:3.0:1787585190.870429:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:3.0:1787585190.870431:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307674658 00000100:00000001:3.0:1787585190.870434:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307678428 00000100:00000001:3.0:1787585190.870436:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307674658 00000100:00000001:3.0:1787585190.870438:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307678428 00000100:00000001:3.0:1787585190.870440:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.870442:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.870445:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.870448:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.870455:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787585190.870458:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1787585190.870460:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.870462:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585190.870463:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787585190.870466:0:2413:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:3.0:1787585190.870468:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.870469:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787585190.870471:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.870472:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.870473:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585190.870474:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787585190.870479:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.870494:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787585190.870496:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.870498:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787585190.870500:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787585190.870502:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787585190.870505:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438016:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.870509:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787585190.870511:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787585190.870513:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.870515:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787585190.870518:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f4605ae00. 02000000:00000001:3.0:1787585190.870520:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585190.870524:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:3.0:1787585190.870526:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f434def78. 00000400:00000010:3.0:1787585190.870530:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd220. 00000100:00000200:3.0:1787585190.870533:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438016, portal 10 00000100:00000001:3.0:1787585190.870534:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787585190.870536:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802458112 : -118680907093504 : ffff940f72b0ea00) 00000100:00000040:3.0:1787585190.870541:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.870548:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787585190.870550:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:3.0:1787585190.870553:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd660. 00000100:00000200:3.0:1787585190.870555:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500438016, offset 0 00000400:00000200:3.0:1787585190.870559:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:3.0:1787585190.870563:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.870567:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:3.0:1787585190.870570:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787585190.870572:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:3.0:1787585190.870574:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787585190.870576:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:3.0:1787585190.870585:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538718:538718:254:4294967295] 192.168.203.132@tcp LPNI seq info [538683:538683:6:4294967295] 00000400:00000200:3.0:1787585190.870589:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.870595:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:3.0:1787585190.870600:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:3.0:1787585190.870603:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940e410e9b00. 00000800:00000200:3.0:1787585190.870607:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:3.0:1787585190.870611:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585190.870614:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940e410e9b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787585190.870617:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.870620:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: using a timeout of 22327 usecs (23 jiffies) 00000400:00000200:3.0:1787585190.870623:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:3.0:1787585190.870625:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307674655 00000100:00000001:3.0:1787585190.870628:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307678428 00000100:00000001:3.0:1787585190.870629:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307674655 00000100:00000001:3.0:1787585190.870632:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307678428 00000100:00000001:3.0:1787585190.870634:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.870636:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.870638:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787585190.870643:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.870650:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787585190.870651:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.870653:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787585190.870655:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787585190.870657:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787585190.870660:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438272:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.870663:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787585190.870666:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787585190.870667:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.870674:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787585190.870677:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f4605be00. 02000000:00000001:3.0:1787585190.871126:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585190.871133:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:3.0:1787585190.871137:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f434de420. 00000400:00000010:3.0:1787585190.871142:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd4c8. 00000100:00000200:3.0:1787585190.871146:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438272, portal 4 00000100:00000001:3.0:1787585190.871148:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787585190.871149:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802455424 : -118680907096192 : ffff940f72b0df80) 00000100:00000040:3.0:1787585190.871157:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.871164:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787585190.871166:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:3.0:1787585190.871169:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd990. 00000100:00000200:3.0:1787585190.871171:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1874405500438272, offset 0 00000400:00000200:3.0:1787585190.871175:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:3.0:1787585190.871179:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.871184:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:3.0:1787585190.871187:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787585190.871190:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:3.0:1787585190.871260:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787585190.871263:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:3.0:1787585190.871267:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538719:538719:253:4294967295] 192.168.203.132@tcp LPNI seq info [538684:538684:5:4294967295] 00000400:00000200:3.0:1787585190.871271:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.871278:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:3.0:1787585190.871283:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:3.0:1787585190.871287:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940e410e9700. 00000800:00000200:3.0:1787585190.871358:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:3.0:1787585190.871363:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585190.871365:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940e410e9700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787585190.871369:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.871379:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: using a timeout of 122631 usecs (123 jiffies) 00000400:00000200:3.0:1787585190.871497:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:3.0:1787585190.871500:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307674757 00000100:00000001:3.0:1787585190.871503:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307678428 00000100:00000001:3.0:1787585190.871505:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307674757 00000100:00000001:3.0:1787585190.871507:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307678428 00000100:00000001:3.0:1787585190.871509:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.871511:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.871515:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.871519:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.871535:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.871537:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1787585190.871576:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585190.871580:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940e410e9f00. 00000400:00000200:3.0:1787585190.871584:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.871590:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585190.871595:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd550 00000400:00000010:3.0:1787585190.871598:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd550. 00000100:00000001:3.0:1787585190.871601:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585190.871606:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.871616:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.871619:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.871626:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.871628:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585190.871639:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585190.871641:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940e410e9a00. 00000400:00000200:3.0:1787585190.871644:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.871647:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585190.871649:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604ddd48 00000400:00000010:3.0:1787585190.871651:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604ddd48. 00000100:00000001:3.0:1787585190.871659:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585190.871663:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.871669:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.871671:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.871677:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.871678:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585190.872141:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585190.872143:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940e410e9b00. 00000400:00000200:3.0:1787585190.872147:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.872152:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585190.872155:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd660 00000400:00000010:3.0:1787585190.872156:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd660. 00000100:00000001:3.0:1787585190.872158:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585190.872162:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.872170:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.872173:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.872179:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.872180:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585190.873278:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585190.873285:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940e410e9700. 00000400:00000200:3.0:1787585190.873349:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.873358:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585190.873364:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd990 00000400:00000010:3.0:1787585190.873366:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd990. 00000100:00000001:3.0:1787585190.873370:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585190.873377:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.873777:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.873791:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.873876:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.873879:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1787585190.874873:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.874887:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585190.874891:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.874894:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.874901:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.874908:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585190.874916:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.874919:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.874925:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.874929:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 25 MB=0x6a8c30a04c580 00000400:00000200:3.0:1787585190.874934:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.874939:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513dcd [1] + 224 00000800:00000001:3.0:1787585190.874946:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.874952:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.874956:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.874959:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585190.874966:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.874972:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585190.874981:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.875007:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585190.875015:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.875018:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.875035:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.875038:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1787585190.875045:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787585190.875047:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787585190.875049:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f434dedc0. 00000400:00000200:3.0:1787585190.875177:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd2a8 00000400:00000010:3.0:1787585190.875179:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd2a8. 00000100:00000001:3.0:1787585190.875182:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.875186:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787585190.875256:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.875262:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787585190.875264:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.875267:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1787585190.875269:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1787585190.875271:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1787585190.875274:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.875277:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1787585190.875283:0:2413:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f60d3c460 time=12 v=5 (5 5 5 5) 00000100:00001000:3.0:1787585190.875287:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f60d3c418 time=55 v=5 (1 1 1 1) 00000100:00000001:3.0:1787585190.875346:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1787585190.875348:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585190.875351:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1787585190.875353:0:2413:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.875356:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.875361:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.875368:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.875371:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787585190.875372:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787585190.875376:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787585190.875463:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2413:1874405500437888:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.875469:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.875471:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.875475:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0c700 x1874405500437888/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.875488:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787585190.875491:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787585190.875494:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f4605ba00. 02000000:00000001:3.0:1787585190.875497:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787585190.875500:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787585190.875502:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f60d3c000 refcount=3 obd=MGC192.168.203.132@tcp 00000020:00000001:3.0:1787585190.875504:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787585190.875507:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9300. 02000000:00000001:3.0:1787585190.875510:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787585190.875512:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787585190.875514:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0c700. 00000100:00000001:3.0:1787585190.875517:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787585190.875518:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.875521:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.875523:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.875525:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585190.875527:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.875528:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1787585190.878025:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.878041:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585190.878045:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.878047:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.878138:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.878145:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585190.878155:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.878160:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.878166:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.878170:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 4 MB=0x6a8c30a04c700 00000400:00000200:3.0:1787585190.878175:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.878180:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513dfd [1] + 224 00000800:00000001:3.0:1787585190.878187:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.878265:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.878269:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.878275:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585190.878351:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.878362:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585190.878374:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.878631:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585190.878645:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.878649:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.878665:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.878669:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1787585190.878675:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787585190.878678:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787585190.879928:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f434de420. 00000400:00000200:3.0:1787585190.879937:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd4c8 00000400:00000010:3.0:1787585190.879939:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd4c8. 00000100:00000001:3.0:1787585190.879945:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.879956:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787585190.879966:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.879973:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787585190.879976:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.879980:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1787585190.879984:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1787585190.879986:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1787585190.879988:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.880001:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1787585190.880011:0:2413:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f48b96c60 time=12 v=5 (5 5 5 5) 00000100:00001000:3.0:1787585190.880016:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f48b96c18 time=109 v=5 (1 1 0 0) 00000100:00000001:3.0:1787585190.880019:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1787585190.880020:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585190.880023:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1787585190.880026:0:2413:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.880030:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1787585190.880041:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-OST0000-osc-ffff940f48b94800: skip recheck: last_committed 42949696454 00000100:00000001:3.0:1787585190.880043:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1787585190.880045:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.881146:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.881162:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.881166:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787585190.881168:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787585190.881175:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787585190.881183:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438272:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.881189:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.881253:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.881258:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0df80 x1874405500438272/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585206 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.881267:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787585190.881269:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787585190.881273:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f4605be00. 02000000:00000001:3.0:1787585190.881277:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787585190.881280:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787585190.881282:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f48b96800 refcount=3 obd=lustre-OST0000-osc-ffff940f48b94800 00000020:00000001:3.0:1787585190.881284:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787585190.881341:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9e00. 02000000:00000001:3.0:1787585190.881346:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787585190.881348:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787585190.881350:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0df80. 00000100:00000001:3.0:1787585190.881353:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787585190.881355:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.881358:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.881360:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.881363:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585190.881365:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.881366:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1787585190.881900:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.881914:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585190.881917:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.881920:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.881927:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.881933:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585190.881941:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.881945:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.881950:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.881954:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04c680 00000400:00000200:3.0:1787585190.881959:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.881964:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513ddd [1] + 224 00000800:00000001:3.0:1787585190.881971:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.881974:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.881977:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.881982:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585190.881989:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.882004:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585190.882014:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.882030:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585190.882038:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.882042:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585190.882043:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.882045:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.882048:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.882480:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585190.882487:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.882490:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.882494:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.882497:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04c600 00000400:00000200:3.0:1787585190.882501:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585190.882510:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513ded [1] + 224 00000800:00000001:3.0:1787585190.882516:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.882524:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585190.882527:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585190.882530:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585190.882534:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.882539:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585190.882547:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.882554:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585190.882558:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585190.882560:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.882572:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.882575:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1787585190.882579:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787585190.882581:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787585190.882583:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f434def78. 00000400:00000200:3.0:1787585190.882587:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd220 00000400:00000010:3.0:1787585190.882589:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd220. 00000100:00000001:3.0:1787585190.882591:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.882595:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787585190.882604:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.882610:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787585190.882612:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.882615:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1787585190.882617:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1787585190.882619:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1787585190.882621:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.882624:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1787585190.882632:0:2413:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482d8460 time=12 v=60 (60 60 60 60) 00000100:00001000:3.0:1787585190.882640:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482d8418 time=55 v=5 (1 1 1 1) 00000100:00000001:3.0:1787585190.882643:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1787585190.882645:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585190.882648:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1787585190.882652:0:2413:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.882655:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1787585190.882657:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff940f48b94800: skip recheck: last_committed 30064771622 00000100:00000001:3.0:1787585190.882658:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1787585190.882660:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.882664:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.882671:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.882673:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787585190.882675:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787585190.884153:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787585190.884179:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438016:192.168.203.132@tcp:400:kworker.0 00000400:00000001:3.0:1787585190.884190:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787585190.884267:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787585190.884271:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f434de840. 00000400:00000200:3.0:1787585190.884275:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd770 00000400:00000010:3.0:1787585190.884278:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd770. 00000100:00000001:3.0:1787585190.884281:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585190.884359:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787585190.884371:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.884378:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787585190.884380:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.884566:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1787585190.884569:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1787585190.884581:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1787585190.884584:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585190.884586:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1787585190.884594:0:2413:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482dc460 time=12 v=60 (60 60 60 60) 00000100:00001000:3.0:1787585190.884598:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482dc418 time=35 v=5 (1 1 1 1) 00000100:00000001:3.0:1787585190.884601:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1787585190.884603:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585190.884605:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1787585190.884610:0:2413:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1787585190.884617:0:2413:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.884619:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1787585190.884621:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff940f48b94800: skip recheck: last_committed 21474836545 00000100:00000001:3.0:1787585190.884623:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1787585190.884625:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585190.884629:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.884636:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585190.884639:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787585190.884641:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787585190.884644:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787585190.884652:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438144:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585190.884656:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.884658:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.884662:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0ea00 x1874405500438016/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.884669:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787585190.884672:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787585190.884675:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f4605ae00. 02000000:00000001:3.0:1787585190.884679:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787585190.885368:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787585190.885370:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f482d8000 refcount=8 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000020:00000001:3.0:1787585190.885374:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787585190.885377:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9900. 02000000:00000001:3.0:1787585190.885759:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787585190.885762:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787585190.885764:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0ea00. 00000100:00000001:3.0:1787585190.885769:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787585190.885770:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.885773:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585190.885780:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0f480 x1874405500438144/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585275 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585190.885788:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787585190.885790:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787585190.885793:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f4605b400. 02000000:00000001:3.0:1787585190.885860:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787585190.885863:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787585190.885864:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f482dc000 refcount=3 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000020:00000001:3.0:1787585190.885867:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787585190.885869:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9100. 02000000:00000001:3.0:1787585190.885871:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787585190.885873:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787585190.885874:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0f480. 00000100:00000001:3.0:1787585190.885877:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787585190.885878:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.885880:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585190.885888:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787585190.885890:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585190.885893:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585190.885895:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585191.309445:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585191.309453:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585191.309459:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585191.309460:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0F:1787585191.310105:0:247673:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1787585191.310162:0:247673:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1787585191.310166:0:247673:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585192.333449:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585192.333458:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585192.333464:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585192.333472:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0F:1787585192.333687:0:241963:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787585192.333697:0:241963:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787585192.333701:0:241963:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585193.357570:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585193.357582:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585193.357589:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585193.357591:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0F:1787585193.357898:0:235982:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585193.357918:0:235982:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585193.357922:0:235982:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585194.384039:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585194.384047:0:279548:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1787585194.384049:0:279548:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1787585194.384052:0:279548:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585194.384057:0:279548:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1787585194.384059:0:279548:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585194.384061:0:279548:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585194.384063:0:279548:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585194.384067:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585194.384069:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787585194.384143:0:247674:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585194.384146:0:247674:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585194.384149:0:247674:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1787585194.701512:0:279548:0:(connection.c:75:cpu_latency_work()) work item of ffff940f44e33b30 (cpu 3) has reached its deadline 4307678428, at 4307678464 00000100:00000010:3.0:1787585194.701530:0:279548:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff940f714cab40. 00000100:00000040:2.0:1787585194.701873:0:265667:0:(connection.c:75:cpu_latency_work()) work item of ffff940f44e33a20 (cpu 2) has reached its deadline 4307678428, at 4307678464 00000100:00000010:2.0:1787585194.701879:0:265667:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff940f714ca180. 00010000:00000001:3.0:1787585195.413714:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585195.413721:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585195.413726:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585195.413728:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787585195.413821:0:235981:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585195.413824:0:235981:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585195.413827:0:235981:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1787585195.981613:0:284577:0:(pinger.c:229:ptlrpc_pinger_process_import()) df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585195.981624:0:284577:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585195.981635:0:284577:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0f480. 00000020:00000040:3.0:1787585195.981640:0:284577:0:(genops.c:1210:class_import_get()) import ffff940f60d3c000 refcount=4 obd=MGC192.168.203.132@tcp 00000100:00000001:3.0:1787585195.981645:0:284577:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585195.981648:0:284577:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585195.981653:0:284577:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585195.981660:0:284577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9900. 00000100:00000001:3.0:1787585195.981666:0:284577:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981677:0:284577:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585195.981689:0:284577:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585195.981692:0:284577:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981699:0:284577:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005cec7c32] to pc [ptlrpcd_01_01+1] req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.981748:0:284577:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981751:0:284577:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585195.981755:0:284577:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585195.981758:0:284577:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0ea00. 00000020:00000040:3.0:1787585195.981760:0:284577:0:(genops.c:1210:class_import_get()) import ffff940f482d8000 refcount=9 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000100:00000001:3.0:1787585195.981763:0:284577:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585195.981764:0:284577:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585195.981766:0:284577:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585195.981769:0:284577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9e00. 00000100:00000001:3.0:1787585195.981772:0:284577:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981776:0:284577:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585195.981782:0:284577:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585195.981784:0:284577:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981788:0:284577:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009afa0799] to pc [ptlrpcd_01_00+0] req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.981795:0:284577:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981797:0:284577:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585195.981800:0:284577:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585195.981802:0:284577:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0df80. 00000020:00000040:3.0:1787585195.981804:0:284577:0:(genops.c:1210:class_import_get()) import ffff940f482dc000 refcount=4 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000100:00000001:3.0:1787585195.981805:0:284577:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585195.981806:0:284577:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585195.981807:0:284577:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585195.981810:0:284577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9300. 00000100:00000001:3.0:1787585195.981812:0:284577:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981816:0:284577:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585195.981834:0:284577:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585195.981841:0:284577:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981846:0:284577:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092ab15fa] to pc [ptlrpcd_01_01+1] req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.981855:0:284577:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981857:0:284577:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585195.981860:0:284577:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585195.981863:0:284577:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0c700. 00000020:00000040:3.0:1787585195.981865:0:284577:0:(genops.c:1210:class_import_get()) import ffff940f48b96800 refcount=4 obd=lustre-OST0000-osc-ffff940f48b94800 00000100:00000001:3.0:1787585195.981868:0:284577:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585195.981869:0:284577:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585195.981871:0:284577:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585195.981874:0:284577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9700. 00000100:00000001:3.0:1787585195.981877:0:284577:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981881:0:284577:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585195.981888:0:284577:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585195.981890:0:284577:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981893:0:284577:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a739ccc1] to pc [ptlrpcd_01_00+0] req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.981898:0:284577:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.981900:0:284577:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787585195.981904:0:284577:0:(pinger.c:247:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787585195.981909:0:284577:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13017) 00000100:00000001:3.0:1787585195.981921:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585195.981924:0:2412:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585195.981926:0:2412:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787585195.981931:0:2412:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.981939:0:2412:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787585195.981942:0:2412:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585195.981953:0:2412:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787585195.981963:0:2412:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787585195.981966:0:2412:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787585195.981970:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fac6ef4a-3c6f-40fa-8410-e2886a870891:2412:1874405500438528:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585195.981974:0:2412:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787585195.981979:0:2412:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787585195.981982:0:2412:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585195.981984:0:2412:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787585195.981988:0:2412:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f4605bc00. 02000000:00000001:3.0:1787585195.981990:0:2412:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585195.982000:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:3.0:1787585195.982004:0:2412:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f434de840. 00000400:00000010:3.0:1787585195.982011:0:2412:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd770. 00000100:00000200:3.0:1787585195.982029:0:2412:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438528, portal 10 00000100:00000001:3.0:1787585195.982032:0:2412:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787585195.982034:0:2412:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802458112 : -118680907093504 : ffff940f72b0ea00) 00000100:00000040:3.0:1787585195.982042:0:2412:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.982049:0:2412:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787585195.982052:0:2412:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:3.0:1787585195.982054:0:2412:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd220. 00000100:00000200:3.0:1787585195.982057:0:2412:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500438528, offset 0 00000400:00000200:3.0:1787585195.982063:0:2412:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:3.0:1787585195.982072:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.982079:0:2412:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:3.0:1787585195.982085:0:2412:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787585195.982089:0:2412:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:3.0:1787585195.982092:0:2412:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787585195.982095:0:2412:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:3.0:1787585195.982098:0:2412:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538720:538720:256:4294967295] 192.168.203.132@tcp LPNI seq info [538685:538685:8:4294967295] 00000400:00000200:3.0:1787585195.982102:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.982111:0:2412:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:3.0:1787585195.982135:0:2412:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:3.0:1787585195.982140:0:2412:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940e410e9b00. 00000800:00000200:3.0:1787585195.982147:0:2412:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:3.0:1787585195.982154:0:2412:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585195.982159:0:2412:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940e410e9b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787585195.982170:0:2412:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.982174:0:2412:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: using a timeout of 22326 usecs (23 jiffies) 00000400:00000200:3.0:1787585195.982178:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:3.0:1787585195.982181:0:2412:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307679767 00000100:00000010:3.0:1787585195.982186:0:2412:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f714cab40. 00000100:00000001:3.0:1787585195.982193:0:2412:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request now active for cpu 2 00000100:00000001:3.0:1787585195.982199:0:2412:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307679767 00000100:00000010:3.0:1787585195.982202:0:2412:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f714ca0c0. 00000100:00000001:3.0:1787585195.982206:0:2412:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request now active for cpu 3 00000100:00000001:3.0:1787585195.982209:0:2412:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585195.982211:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585195.982212:0:2412:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.982215:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585195.982217:0:2412:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787585195.982219:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787585195.982224:0:2412:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787585195.982230:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.982234:0:2412:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787585195.982238:0:2412:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585195.982243:0:2412:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787585195.982247:0:2412:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585195.982247:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787585195.982250:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585195.982252:0:2412:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585195.982255:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787585195.982259:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787585195.982260:0:2412:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fac6ef4a-3c6f-40fa-8410-e2886a870891:2412:1874405500438784:192.168.203.132@tcp:400:kworker.0 02000000:00000001:2.0:1787585195.982265:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.982267:0:2412:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787585195.982272:0:2412:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1787585195.982272:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2413:1874405500438400:192.168.203.132@tcp:400:kworker.0 02000000:00000001:3.0:1787585195.982275:0:2412:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585195.982281:0:2412:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787585195.982282:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1787585195.982285:0:2412:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f4605b800. 02000000:00000001:2.0:1787585195.982287:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787585195.982289:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585195.982293:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787585195.982295:0:2412:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787585195.982299:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f43452c00. 00000400:00000200:3.0:1787585195.982303:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 02000000:00000001:2.0:1787585195.982304:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1787585195.982308:0:2412:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f434def78. 00000400:00000200:2.0:1787585195.982313:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:3.0:1787585195.982316:0:2412:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd4c8. 00000400:00000010:2.0:1787585195.982319:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f7358d630. 00000100:00000200:3.0:1787585195.982322:0:2412:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438784, portal 4 00000100:00000001:3.0:1787585195.982326:0:2412:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000400:00000010:2.0:1787585195.982327:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de000. 00000100:00000001:3.0:1787585195.982328:0:2412:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802449152 : -118680907102464 : ffff940f72b0c700) 00000100:00000200:2.0:1787585195.982334:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438400, portal 25 00000100:00000040:3.0:1787585195.982339:0:2412:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.982340:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787585195.982343:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802460800 : -118680907090816 : ffff940f72b0f480) 00000100:00000001:3.0:1787585195.982352:0:2412:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787585195.982354:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585195.982357:0:2412:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:3.0:1787585195.982363:0:2412:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f604dd2a8. 00000100:00000200:3.0:1787585195.982367:0:2412:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1874405500438784, offset 0 00000100:00000001:2.0:1787585195.982368:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787585195.982375:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000200:3.0:1787585195.982379:0:2412:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000010:2.0:1787585195.982381:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de3b8. 00000400:00000200:3.0:1787585195.982519:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.982523:0:2412:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:3.0:1787585195.982526:0:2412:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787585195.982529:0:2412:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:3.0:1787585195.982531:0:2412:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787585195.982533:0:2412:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:3.0:1787585195.982536:0:2412:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538721:538721:255:4294967295] 192.168.203.132@tcp LPNI seq info [538686:538686:7:4294967295] 00000400:00000200:3.0:1787585195.982540:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.982545:0:2412:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:3.0:1787585195.982550:0:2412:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:3.0:1787585195.982554:0:2412:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940e410e9a00. 00000800:00000200:3.0:1787585195.982558:0:2412:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:3.0:1787585195.982562:0:2412:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585195.982565:0:2412:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940e410e9a00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787585195.982568:0:2412:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585195.982571:0:2412:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: using a timeout of 122555 usecs (123 jiffies) 00000400:00000200:3.0:1787585195.982574:0:2412:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:3.0:1787585195.982582:0:2412:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307679868 00000100:00000001:3.0:1787585195.982588:0:2412:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 2, simply extend its deadline from 4307679767 00000100:00000001:3.0:1787585195.982591:0:2412:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307679868 00000100:00000001:3.0:1787585195.982594:0:2412:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 3, simply extend its deadline from 4307679767 00000100:00000001:3.0:1787585195.982596:0:2412:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585195.982598:0:2412:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.982601:0:2412:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.982603:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585195.982610:0:2412:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787585195.982612:0:2412:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1787585195.982614:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585195.982615:0:2412:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787585195.982616:0:2412:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.982618:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1787585195.982658:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585195.982662:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940e410e9b00. 00000400:00000200:3.0:1787585195.982668:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585195.982673:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000200:2.0:1787585195.983341:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1874405500438400, offset 0 00000400:00000200:2.0:1787585195.983362:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:2.0:1787585195.983368:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585195.983373:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:2.0:1787585195.983377:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787585195.983381:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:2.0:1787585195.983407:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787585195.983409:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:2.0:1787585195.983412:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538722:538722:254:4294967295] 192.168.203.132@tcp LPNI seq info [538687:538687:6:4294967295] 00000400:00000200:2.0:1787585195.983472:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585195.983480:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:2.0:1787585195.983486:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:2.0:1787585195.983491:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f4347df00. 00000800:00000200:2.0:1787585195.983497:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:2.0:1787585195.983502:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:2.0:1787585195.983506:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f4347df00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787585195.983510:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585195.983514:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:2.0:1787585195.983516:0:2413:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 2 is 4307683541 00000100:00000001:2.0:1787585195.983521:0:2413:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4307679868 00000100:00000001:2.0:1787585195.983525:0:2413:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 3 is 4307683541 00000100:00000001:2.0:1787585195.983528:0:2413:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4307679868 00000100:00000001:2.0:1787585195.983531:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585195.983534:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.983537:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787585195.983541:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.983549:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787585195.983551:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585195.983555:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787585195.983557:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787585195.983565:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787585195.983569:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438656:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585195.983573:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787585195.983576:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787585195.983579:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585195.983581:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787585195.983585:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f43453400. 02000000:00000001:2.0:1787585195.983586:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585195.983590:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:2.0:1787585195.983593:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f7358d000. 00000400:00000010:2.0:1787585195.983598:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de330. 00000100:00000200:2.0:1787585195.983602:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438656, portal 10 00000100:00000001:2.0:1787585195.983604:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787585195.983605:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802455424 : -118680907096192 : ffff940f72b0df80) 00000100:00000040:2.0:1787585195.983611:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.983618:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787585195.983620:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:2.0:1787585195.983622:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de660. 00000100:00000200:2.0:1787585195.983624:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500438656, offset 0 00000400:00000200:2.0:1787585195.983629:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:2.0:1787585195.983633:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585195.983636:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:2.0:1787585195.983639:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787585195.983641:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:2.0:1787585195.983643:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787585195.983646:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:2.0:1787585195.983648:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538723:538723:253:4294967295] 192.168.203.132@tcp LPNI seq info [538688:538688:5:4294967295] 00000400:00000200:2.0:1787585195.983652:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585195.983657:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:2.0:1787585195.983662:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:2.0:1787585195.983671:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f4347d500. 00000800:00000200:2.0:1787585195.983675:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:2.0:1787585195.983678:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:2.0:1787585195.983681:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f4347d500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787585195.983756:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585195.983761:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: using a timeout of 25401 usecs (26 jiffies) 00000400:00000200:2.0:1787585195.983765:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:2.0:1787585195.983767:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307679771 00000100:00000001:2.0:1787585195.983769:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307683541 00000100:00000001:2.0:1787585195.983771:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307679771 00000100:00000001:2.0:1787585195.983773:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307683541 00000100:00000001:2.0:1787585195.983775:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585195.983777:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.983779:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.983782:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585195.983787:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787585195.983789:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1787585195.983792:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585195.983793:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585195.983795:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.983797:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585195.985721:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd220 00000400:00000010:3.0:1787585195.985729:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd220. 00000100:00000001:3.0:1787585195.985738:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585195.985749:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.985762:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585195.985766:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.985787:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.985789:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585195.985994:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585195.985998:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940e410e9a00. 00000400:00000200:3.0:1787585195.986004:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585195.986009:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585195.986013:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd2a8 00000400:00000010:3.0:1787585195.986040:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd2a8. 00000100:00000001:3.0:1787585195.986043:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585195.986048:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.986055:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585195.986058:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.986063:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.986065:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585195.986075:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585195.986078:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f4347df00. 00000400:00000200:3.0:1787585195.986081:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585195.986085:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585195.986087:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de3b8 00000400:00000010:3.0:1787585195.986089:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de3b8. 00000100:00000001:3.0:1787585195.986091:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585195.986095:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.986101:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585195.986104:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.986108:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.986110:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585195.986240:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585195.986243:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f4347d500. 00000400:00000200:3.0:1787585195.986246:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585195.986250:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585195.986253:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de660 00000400:00000010:3.0:1787585195.986255:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de660. 00000100:00000001:3.0:1787585195.986257:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585195.986261:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.986269:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585195.986278:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.986284:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585195.986285:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1787585195.992929:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.992946:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585195.992951:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.992962:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585195.992980:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.992988:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585195.992998:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993003:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993008:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993012:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 4 MB=0x6a8c30a04c900 00000400:00000200:3.0:1787585195.993017:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993024:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e1d [1] + 224 00000800:00000001:3.0:1787585195.993032:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.993036:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585195.993040:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585195.993045:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585195.993053:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585195.993063:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585195.993075:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.993097:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585195.993106:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.993109:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585195.993111:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.993113:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585195.993116:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993120:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585195.993138:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993142:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993145:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993148:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 25 MB=0x6a8c30a04c780 00000400:00000200:3.0:1787585195.993152:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585195.993156:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e25 [1] + 224 00000800:00000001:3.0:1787585195.993160:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.993166:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585195.993168:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585195.993172:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585195.993175:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585195.993180:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585195.993188:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585195.993199:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585195.993203:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585195.993206:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585195.993837:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585195.993843:0:2412:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1787585195.993851:0:2412:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585195.993853:0:2412:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585195.993856:0:2412:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f434def78. 00000400:00000200:2.0:1787585195.993862:0:2412:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd4c8 00000400:00000010:2.0:1787585195.993864:0:2412:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd4c8. 00000100:00000001:2.0:1787585195.993867:0:2412:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585195.993874:0:2412:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585195.993883:0:2412:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.993903:0:2412:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585195.993906:0:2412:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585195.993908:0:2412:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585195.993911:0:2412:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585195.993914:0:2412:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585195.993916:0:2412:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585195.993919:0:2412:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585195.993928:0:2412:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f48b96c60 time=17 v=5 (5 5 5 5) 00000100:00001000:2.0:1787585195.993932:0:2412:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f48b96c18 time=114 v=5 (1 1 0 0) 00000100:00000001:2.0:1787585195.993935:0:2412:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585195.993937:0:2412:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585195.993939:0:2412:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1787585195.993942:0:2412:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.993946:0:2412:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1787585195.993949:0:2412:0:(client.c:3121:ptlrpc_free_committed()) lustre-OST0000-osc-ffff940f48b94800: skip recheck: last_committed 42949696454 00000100:00000001:2.0:1787585195.993951:0:2412:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1787585195.993953:0:2412:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585195.993968:0:2412:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.993976:0:2412:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585195.993978:0:2412:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585195.993981:0:2412:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585195.993985:0:2412:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585195.993992:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fac6ef4a-3c6f-40fa-8410-e2886a870891:2412:1874405500438784:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585195.993997:0:2412:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994000:0:2412:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585195.994003:0:2412:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0c700 x1874405500438784/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.994010:0:2412:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585195.994012:0:2412:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585195.994021:0:2412:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f4605b800. 02000000:00000001:2.0:1787585195.994024:0:2412:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585195.994027:0:2412:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585195.994029:0:2412:0:(genops.c:1223:class_import_put()) import ffff940f48b96800 refcount=3 obd=lustre-OST0000-osc-ffff940f48b94800 00000020:00000001:2.0:1787585195.994031:0:2412:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585195.994033:0:2412:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9700. 02000000:00000001:2.0:1787585195.994037:0:2412:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585195.994038:0:2412:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585195.994040:0:2412:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0c700. 00000100:00000001:2.0:1787585195.994043:0:2412:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585195.994044:0:2412:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585195.994047:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994049:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585195.994050:0:2412:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585195.994052:0:2412:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994053:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994066:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585195.994068:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1787585195.994071:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585195.994073:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585195.994075:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f7358d630. 00000400:00000200:2.0:1787585195.994078:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de000 00000400:00000010:2.0:1787585195.994080:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de000. 00000100:00000001:2.0:1787585195.994082:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585195.994087:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585195.994096:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.994102:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585195.994103:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585195.994104:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585195.994106:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585195.994107:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585195.994108:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585195.994110:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585195.994113:0:2413:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f60d3c460 time=17 v=5 (5 5 5 5) 00000100:00001000:2.0:1787585195.994116:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f60d3c418 time=60 v=5 (1 1 1 1) 00000100:00000001:2.0:1787585195.994124:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585195.994126:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585195.994127:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1787585195.994129:0:2413:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994131:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585195.994136:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.994142:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585195.994144:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585195.994145:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585195.994148:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585195.994154:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2413:1874405500438400:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585195.994158:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994160:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585195.994163:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0f480 x1874405500438400/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585211 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585195.994168:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585195.994169:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585195.994171:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f43452c00. 02000000:00000001:2.0:1787585195.994173:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585195.994174:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585195.994176:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f60d3c000 refcount=3 obd=MGC192.168.203.132@tcp 00000020:00000001:2.0:1787585195.994177:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585195.994179:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9900. 02000000:00000001:2.0:1787585195.994181:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585195.994182:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585195.994184:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0f480. 00000100:00000001:2.0:1787585195.994187:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585195.994188:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585195.994189:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994191:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585195.994192:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585195.994198:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585195.994201:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1787585196.000548:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000572:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585196.000578:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000581:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585196.000593:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000603:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585196.000614:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000620:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000626:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000630:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04c800 00000400:00000200:3.0:1787585196.000636:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000644:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e0d [1] + 224 00000800:00000001:3.0:1787585196.000653:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000677:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585196.000682:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585196.000688:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585196.000698:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585196.000711:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585196.000724:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585196.000751:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585196.000771:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000776:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585196.000778:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000780:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585196.000784:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000787:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585196.000792:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000794:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000797:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000800:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04c880 00000400:00000200:3.0:1787585196.000804:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585196.000809:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e3d [1] + 224 00000800:00000001:3.0:1787585196.000813:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000820:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585196.000823:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585196.000825:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585196.000828:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585196.000832:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585196.000840:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585196.000854:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585196.000865:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585196.000868:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585196.001267:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585196.001274:0:2412:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1787585196.001284:0:2412:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585196.001286:0:2412:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585196.001291:0:2412:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f434de840. 00000400:00000200:2.0:1787585196.001298:0:2412:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f604dd770 00000400:00000010:2.0:1787585196.001301:0:2412:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f604dd770. 00000100:00000001:2.0:1787585196.001304:0:2412:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585196.001311:0:2412:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585196.001320:0:2412:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585196.001327:0:2412:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585196.001330:0:2412:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585196.001333:0:2412:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585196.001337:0:2412:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585196.001340:0:2412:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585196.001343:0:2412:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585196.001346:0:2412:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585196.001370:0:2412:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482d8460 time=18 v=60 (60 60 60 60) 00000100:00001000:2.0:1787585196.001375:0:2412:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482d8418 time=61 v=5 (1 1 1 1) 00000100:00000001:2.0:1787585196.001378:0:2412:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585196.001381:0:2412:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585196.001572:0:2412:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1787585196.001577:0:2412:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585196.001581:0:2412:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1787585196.001584:0:2412:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff940f48b94800: skip recheck: last_committed 30064771622 00000100:00000001:2.0:1787585196.001587:0:2412:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1787585196.001589:0:2412:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585196.001595:0:2412:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585196.001603:0:2412:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585196.001607:0:2412:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585196.001619:0:2412:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585196.001624:0:2412:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585196.001633:0:2412:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fac6ef4a-3c6f-40fa-8410-e2886a870891:2412:1874405500438528:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585196.001638:0:2412:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585196.001641:0:2412:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585196.001645:0:2412:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0ea00 x1874405500438528/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585196.001652:0:2412:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585196.001655:0:2412:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585196.001660:0:2412:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f4605bc00. 02000000:00000001:2.0:1787585196.001664:0:2412:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585196.001668:0:2412:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585196.001669:0:2412:0:(genops.c:1223:class_import_put()) import ffff940f482d8000 refcount=8 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000020:00000001:2.0:1787585196.001672:0:2412:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585196.001675:0:2412:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9e00. 02000000:00000001:2.0:1787585196.001679:0:2412:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585196.001681:0:2412:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585196.001683:0:2412:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0ea00. 00000100:00000001:2.0:1787585196.001687:0:2412:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585196.001689:0:2412:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585196.001691:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585196.001701:0:2412:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787585196.001703:0:2412:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585196.001706:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585196.001708:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585196.002540:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585196.002547:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1787585196.002557:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787585196.002559:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787585196.002563:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f7358d000. 00000400:00000200:3.0:1787585196.002569:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de330 00000400:00000010:3.0:1787585196.002572:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de330. 00000100:00000001:3.0:1787585196.002576:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585196.002586:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787585196.002597:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585196.002605:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787585196.002606:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585196.002608:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:3.0:1787585196.002611:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1787585196.002614:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1787585196.002616:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787585196.002619:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1787585196.002628:0:2413:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482dc460 time=18 v=60 (60 60 60 60) 00000100:00001000:3.0:1787585196.002631:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482dc418 time=41 v=5 (1 1 1 1) 00000100:00000001:3.0:1787585196.002634:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1787585196.002635:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585196.002638:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:3.0:1787585196.002641:0:2413:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1787585196.002649:0:2413:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585196.002652:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:3.0:1787585196.002654:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff940f48b94800: skip recheck: last_committed 21474836545 00000100:00000001:3.0:1787585196.002656:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1787585196.002658:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585196.002662:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585196.002690:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787585196.002693:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787585196.002695:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787585196.002699:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787585196.002706:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500438656:192.168.203.132@tcp:400:kworker.0 00000100:00000001:3.0:1787585196.002711:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585196.002714:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585196.002718:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0df80 x1874405500438656/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585280 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585196.002724:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787585196.002727:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787585196.002730:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f43453400. 02000000:00000001:3.0:1787585196.002733:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787585196.002736:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787585196.002738:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f482dc000 refcount=3 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000020:00000001:3.0:1787585196.002740:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787585196.002742:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9300. 02000000:00000001:3.0:1787585196.002745:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787585196.002746:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787585196.002748:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0df80. 00000100:00000001:3.0:1787585196.002754:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787585196.002755:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585196.002757:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787585196.002774:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787585196.002776:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585196.002779:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787585196.002781:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585196.429550:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585196.429558:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585196.429566:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585196.429567:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787585196.430059:0:247673:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1787585196.430064:0:247673:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1787585196.430068:0:247673:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585197.456773:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585197.456782:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585197.456789:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585197.456790:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787585197.456970:0:241963:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787585197.456977:0:241963:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787585197.456980:0:241963:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585198.479380:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585198.479630:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585198.479637:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585198.479638:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787585198.483945:0:235982:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1787585198.483952:0:235982:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1787585198.483958:0:235982:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1787585198.849526:0:284694:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585199.501474:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585199.501482:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585199.501487:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585199.501488:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787585199.503195:0:247674:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585199.503201:0:247674:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585199.503206:0:247674:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1787585199.821487:0:279548:0:(connection.c:75:cpu_latency_work()) work item of ffff940f44e33b30 (cpu 3) has reached its deadline 4307683541, at 4307683584 00000100:00000010:3.0:1787585199.821502:0:279548:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff940f714ca0c0. 00000100:00000040:2.0:1787585199.822567:0:280011:0:(connection.c:75:cpu_latency_work()) work item of ffff940f44e33a20 (cpu 2) has reached its deadline 4307683541, at 4307683585 00000100:00000010:2.0:1787585199.822577:0:280011:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff940f714cab40. 00010000:00000001:3.0:1787585200.525509:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585200.525517:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585200.525523:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585200.525527:0:279548:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1787585200.525529:0:279548:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1787585200.525535:0:279548:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787585200.525537:0:279548:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787585200.525539:0:279548:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585200.525542:0:279548:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1787585200.525544:0:279548:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585200.525546:0:279548:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585200.525548:0:279548:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787585200.525715:0:235981:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585200.525718:0:235981:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585200.525722:0:235981:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1787585201.101867:0:266992:0:(pinger.c:229:ptlrpc_pinger_process_import()) df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585201.101880:0:266992:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585201.101890:0:266992:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0df80. 00000020:00000040:3.0:1787585201.101896:0:266992:0:(genops.c:1210:class_import_get()) import ffff940f60d3c000 refcount=4 obd=MGC192.168.203.132@tcp 00000100:00000001:3.0:1787585201.101900:0:266992:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585201.101904:0:266992:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585201.101908:0:266992:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585201.101916:0:266992:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9300. 00000100:00000001:3.0:1787585201.101932:0:266992:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.101946:0:266992:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585201.101972:0:266992:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585201.101976:0:266992:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.101983:0:266992:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000092ab15fa] to pc [ptlrpcd_01_01+1] req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.102016:0:266992:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102019:0:266992:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585201.102024:0:266992:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585201.102027:0:266992:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0ce00. 00000020:00000040:3.0:1787585201.102030:0:266992:0:(genops.c:1210:class_import_get()) import ffff940f482d8000 refcount=9 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000100:00000001:3.0:1787585201.102033:0:266992:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585201.102034:0:266992:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585201.102036:0:266992:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585201.102040:0:266992:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9a00. 00000100:00000001:3.0:1787585201.102044:0:266992:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102048:0:266992:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585201.102056:0:266992:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585201.102058:0:266992:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102062:0:266992:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000090df7d63] to pc [ptlrpcd_01_00+0] req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.102076:0:266992:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102078:0:266992:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585201.102082:0:266992:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585201.102085:0:266992:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0f800. 00000020:00000040:3.0:1787585201.102087:0:266992:0:(genops.c:1210:class_import_get()) import ffff940f482dc000 refcount=4 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000100:00000001:3.0:1787585201.102089:0:266992:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585201.102090:0:266992:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585201.102092:0:266992:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585201.102095:0:266992:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9b00. 00000100:00000001:3.0:1787585201.102106:0:266992:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102110:0:266992:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585201.102117:0:266992:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585201.102118:0:266992:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102122:0:266992:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f2a71496] to pc [ptlrpcd_01_01+1] req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.102129:0:266992:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102132:0:266992:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787585201.102135:0:266992:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787585201.102137:0:266992:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f72b0dc00. 00000020:00000040:3.0:1787585201.102140:0:266992:0:(genops.c:1210:class_import_get()) import ffff940f48b96800 refcount=4 obd=lustre-OST0000-osc-ffff940f48b94800 00000100:00000001:3.0:1787585201.102142:0:266992:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787585201.102143:0:266992:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787585201.102145:0:266992:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787585201.102148:0:266992:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940e410e9f00. 00000100:00000001:3.0:1787585201.102151:0:266992:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102155:0:266992:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787585201.102161:0:266992:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787585201.102163:0:266992:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102167:0:266992:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000076b584ad] to pc [ptlrpcd_01_00+0] req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.102174:0:266992:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787585201.102176:0:266992:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787585201.102180:0:266992:0:(pinger.c:247:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787585201.102184:0:266992:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13022) 00000100:00000001:2.0:1787585201.102246:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.102250:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585201.102252:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787585201.102259:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.102267:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787585201.102269:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.102272:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787585201.102275:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787585201.102278:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787585201.102281:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2413:1874405500438912:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585201.102285:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787585201.102290:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787585201.102293:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.102295:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787585201.102299:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f440f9400. 02000000:00000001:2.0:1787585201.102301:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585201.102308:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:2.0:1787585201.102313:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f7358d630. 00000400:00000010:2.0:1787585201.102319:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de000. 00000100:00000200:2.0:1787585201.102326:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500438912, portal 25 00000100:00000001:2.0:1787585201.102329:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787585201.102330:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802455424 : -118680907096192 : ffff940f72b0df80) 00000100:00000040:2.0:1787585201.102336:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.102343:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787585201.102346:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:2.0:1787585201.102348:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488deee0. 00000100:00000200:2.0:1787585201.102351:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1874405500438912, offset 0 00000400:00000200:2.0:1787585201.102357:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:2.0:1787585201.102366:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.102373:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:2.0:1787585201.102379:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787585201.102420:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:2.0:1787585201.102424:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787585201.102427:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:2.0:1787585201.102431:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538724:538724:256:4294967295] 192.168.203.132@tcp LPNI seq info [538689:538689:8:4294967295] 00000400:00000200:2.0:1787585201.102435:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.102444:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:2.0:1787585201.102450:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:2.0:1787585201.102455:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f4347d700. 00000800:00000200:2.0:1787585201.102462:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:2.0:1787585201.102470:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:2.0:1787585201.102474:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f4347d700 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787585201.102490:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585201.102494:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:2.0:1787585201.102496:0:2413:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 2 is 4307688661 00000100:00000010:2.0:1787585201.102501:0:2413:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f715ba4e0. 00000100:00000001:2.0:1787585201.102507:0:2413:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request now active for cpu 2 00000100:00000001:2.0:1787585201.102512:0:2413:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 3 is 4307688661 00000100:00000010:2.0:1787585201.102515:0:2413:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f715ba240. 00000100:00000001:2.0:1787585201.102518:0:2413:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request now active for cpu 3 00000100:00000001:2.0:1787585201.102520:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.102523:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.102526:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787585201.102530:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.102536:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787585201.102537:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.102539:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787585201.102541:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787585201.102542:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787585201.102545:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500439168:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585201.102554:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787585201.102557:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000800:00000200:3.0:1787585201.102783:0:2405:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585201.102788:0:2405:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f4347d700. 00000400:00000200:3.0:1787585201.102797:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585201.102803:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585201.102809:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488deee0 00000400:00000010:3.0:1787585201.102811:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488deee0. 00000100:00000001:3.0:1787585201.102815:0:2405:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585201.102819:0:2405:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.102828:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585201.102831:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.102838:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585201.102840:0:2405:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787585201.103207:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.103210:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787585201.103223:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f440f9a00. 02000000:00000001:2.0:1787585201.103226:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585201.103231:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:2.0:1787585201.103235:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f7358d210. 00000400:00000010:2.0:1787585201.103240:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488deee0. 00000100:00000200:2.0:1787585201.103244:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439168, portal 10 00000100:00000001:2.0:1787585201.103246:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787585201.103247:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802461696 : -118680907089920 : ffff940f72b0f800) 00000100:00000040:2.0:1787585201.103256:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.103263:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787585201.103264:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:2.0:1787585201.103267:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de330. 00000100:00000200:2.0:1787585201.103270:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500439168, offset 0 00000400:00000200:2.0:1787585201.103274:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:2.0:1787585201.103279:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.103283:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:2.0:1787585201.103286:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787585201.103289:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:2.0:1787585201.103291:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787585201.103293:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:2.0:1787585201.103295:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538725:538725:256:4294967295] 192.168.203.132@tcp LPNI seq info [538690:538690:8:4294967295] 00000400:00000200:2.0:1787585201.103299:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.103305:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:2.0:1787585201.103310:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:2.0:1787585201.103313:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f4347d800. 00000800:00000200:2.0:1787585201.103318:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:2.0:1787585201.103329:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:2.0:1787585201.103331:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f4347d800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787585201.103346:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.103350:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: using a timeout of 25401 usecs (26 jiffies) 00000400:00000200:2.0:1787585201.103353:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:2.0:1787585201.103356:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307684891 00000100:00000001:2.0:1787585201.103358:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307688661 00000100:00000001:2.0:1787585201.103360:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307684891 00000100:00000001:2.0:1787585201.103362:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307688661 00000100:00000001:2.0:1787585201.103364:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.103370:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.103373:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.103376:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.103405:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787585201.103408:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1787585201.103410:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.103412:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585201.103413:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787585201.103416:0:2413:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:2.0:1787585201.103418:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.103420:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787585201.103421:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.103423:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.103424:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585201.103425:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787585201.103430:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.103436:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787585201.103438:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.103440:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787585201.103442:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787585201.103444:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787585201.103447:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500439040:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585201.103456:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787585201.103459:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787585201.103461:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.103462:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787585201.103465:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f440f9600. 02000000:00000001:2.0:1787585201.103467:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585201.103471:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:2.0:1787585201.103474:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f7358de18. 00000400:00000010:2.0:1787585201.103478:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de660. 00000100:00000200:2.0:1787585201.103481:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439040, portal 10 00000100:00000001:2.0:1787585201.103482:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787585201.103484:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802450944 : -118680907100672 : ffff940f72b0ce00) 00000100:00000040:2.0:1787585201.103490:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.103496:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787585201.103499:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:2.0:1787585201.103501:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de3b8. 00000100:00000200:2.0:1787585201.103503:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500439040, offset 0 00000400:00000200:2.0:1787585201.103507:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:2.0:1787585201.103511:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.103514:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:2.0:1787585201.103517:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787585201.103520:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:2.0:1787585201.103522:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787585201.103524:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:2.0:1787585201.103526:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538726:538726:255:4294967295] 192.168.203.132@tcp LPNI seq info [538691:538691:7:4294967295] 00000400:00000200:2.0:1787585201.103530:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.103535:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:2.0:1787585201.103539:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:2.0:1787585201.103542:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f4347de00. 00000800:00000200:2.0:1787585201.103546:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:2.0:1787585201.103556:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:3.0:1787585201.103691:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585201.103694:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f4347d800. 00000400:00000200:3.0:1787585201.103698:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585201.103702:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585201.103706:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de330 00000400:00000010:3.0:1787585201.103708:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de330. 00000100:00000001:3.0:1787585201.103710:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585201.103715:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.103724:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585201.103726:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.103744:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585201.103745:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:2.0:1787585201.103987:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f4347de00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787585201.103998:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.104002:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: using a timeout of 22327 usecs (23 jiffies) 00000400:00000200:2.0:1787585201.104005:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:2.0:1787585201.104007:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307684889 00000100:00000001:2.0:1787585201.104010:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307688661 00000100:00000001:2.0:1787585201.104012:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307684889 00000100:00000001:2.0:1787585201.104014:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307688661 00000100:00000001:2.0:1787585201.104016:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.104018:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.104021:0:2413:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787585201.104025:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.104033:0:2413:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787585201.104035:0:2413:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.104037:0:2413:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787585201.104039:0:2413:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787585201.104041:0:2413:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787585201.104044:0:2413:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f72b0dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500439296:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585201.104047:0:2413:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787585201.104050:0:2413:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000800:00000200:3.0:1787585201.104051:0:2405:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000001:2.0:1787585201.104051:0:2413:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1787585201.104054:0:2405:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f4347de00. 02000000:00000001:2.0:1787585201.104054:0:2413:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1787585201.104057:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:2.0:1787585201.104057:0:2413:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f440f9e00. 02000000:00000001:2.0:1787585201.104059:0:2413:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585201.104061:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:2.0:1787585201.104063:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.104064:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de3b8 00000400:00000010:2.0:1787585201.104065:0:2413:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f7358d0b0. 00000400:00000010:3.0:1787585201.104066:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de3b8. 00000100:00000001:3.0:1787585201.104069:0:2405:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:2.0:1787585201.104070:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de2a8. 00000100:00000200:2.0:1787585201.104073:0:2413:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439296, portal 4 00000100:00000001:2.0:1787585201.104075:0:2413:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000200:3.0:1787585201.104078:0:2405:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.104080:0:2413:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392802454528 : -118680907097088 : ffff940f72b0dc00) 00000100:00000040:2.0:1787585201.104086:0:2413:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.104093:0:2413:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787585201.104094:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585201.104095:0:2413:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000100:00000040:3.0:1787585201.104097:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787585201.104097:0:2413:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f488de088. 00000100:00000200:2.0:1787585201.104100:0:2413:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1874405500439296, offset 0 00000100:00000001:3.0:1787585201.104104:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787585201.104104:0:2413:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000100:00000001:3.0:1787585201.104106:0:2405:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:2.0:1787585201.104109:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.104112:0:2413:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:2.0:1787585201.104115:0:2413:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787585201.104117:0:2413:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:2.0:1787585201.104120:0:2413:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787585201.104122:0:2413:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:2.0:1787585201.104125:0:2413:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538727:538727:255:4294967295] 192.168.203.132@tcp LPNI seq info [538692:538692:7:4294967295] 00000400:00000200:2.0:1787585201.104128:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585201.104134:0:2413:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:2.0:1787585201.104138:0:2413:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:2.0:1787585201.104141:0:2413:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f4347d200. 00000800:00000200:2.0:1787585201.104145:0:2413:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:2.0:1787585201.104150:0:2413:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:2.0:1787585201.104152:0:2413:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f4347d200 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787585201.104160:0:2413:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.104164:0:2413:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: using a timeout of 122480 usecs (123 jiffies) 00000400:00000200:2.0:1787585201.104167:0:2413:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:2.0:1787585201.104169:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307684989 00000100:00000001:2.0:1787585201.104172:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307688661 00000100:00000001:2.0:1787585201.104173:0:2413:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307684989 00000100:00000001:2.0:1787585201.104176:0:2413:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307688661 00000800:00000200:3.0:1787585201.104181:0:2404:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585201.104183:0:2404:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f4347d200. 00000100:00000001:2.0:1787585201.104183:0:2413:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.104185:0:2413:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585201.104186:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1787585201.104188:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585201.104189:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:2.0:1787585201.104191:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787585201.104192:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de088 00000400:00000010:3.0:1787585201.104194:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de088. 00000100:00000001:3.0:1787585201.104196:0:2404:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585201.104200:0:2404:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.104202:0:2412:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.104204:0:2412:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585201.104207:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585201.104209:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.104215:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585201.104217:0:2404:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:3.0:1787585201.117993:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118012:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585201.118017:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118019:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.118028:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118036:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585201.118045:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118050:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118056:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118060:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04ca00 00000400:00000200:3.0:1787585201.118064:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118086:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e6d [1] + 224 00000800:00000001:3.0:1787585201.118094:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118099:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.118102:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585201.118106:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585201.118114:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585201.118124:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585201.118135:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.118157:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585201.118166:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118169:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585201.118171:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118173:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.118176:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118179:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585201.118183:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118185:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118188:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118190:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 4 MB=0x6a8c30a04cb00 00000400:00000200:3.0:1787585201.118194:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118198:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e7d [1] + 224 00000800:00000001:3.0:1787585201.118202:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118205:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.118208:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585201.118211:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585201.118214:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585201.118218:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585201.118232:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.118244:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585201.118249:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118252:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585201.118254:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118256:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.118259:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118262:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585201.118266:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118269:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118272:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118274:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04ca80 00000400:00000200:3.0:1787585201.118278:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.118281:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e5d [1] + 224 00000800:00000001:3.0:1787585201.118285:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118291:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.118293:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585201.118307:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585201.118310:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585201.118314:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585201.118323:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.118330:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585201.118334:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.118337:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.118866:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.118871:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1787585201.118877:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585201.118880:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585201.118883:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f7358de18. 00000400:00000200:2.0:1787585201.118887:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de660 00000400:00000010:2.0:1787585201.118890:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de660. 00000100:00000001:2.0:1787585201.118893:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585201.118899:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585201.118908:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.118915:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585201.118917:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.118930:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585201.118933:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585201.118936:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585201.118938:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.118941:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585201.118950:0:2413:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482d8460 time=23 v=60 (60 60 60 60) 00000100:00001000:2.0:1787585201.118955:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482d8418 time=66 v=5 (1 1 1 1) 00000100:00000001:2.0:1787585201.118957:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585201.118959:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585201.118961:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1787585201.118964:0:2413:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.118968:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1787585201.118970:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff940f48b94800: skip recheck: last_committed 30064771622 00000100:00000001:2.0:1787585201.118972:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1787585201.118973:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.118987:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.118994:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.118997:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585201.119000:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585201.119003:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585201.119010:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500439040:192.168.203.132@tcp:400:kworker.0 00000400:00000001:2.0:1787585201.119016:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585201.119018:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585201.119020:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f7358d0b0. 00000400:00000200:2.0:1787585201.119022:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de2a8 00000400:00000010:2.0:1787585201.119024:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de2a8. 00000100:00000001:2.0:1787585201.119026:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585201.119030:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585201.119038:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119045:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585201.119046:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.119048:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585201.119049:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585201.119050:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585201.119052:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.119054:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585201.119059:0:2413:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f48b96c60 time=23 v=5 (5 5 5 5) 00000100:00001000:2.0:1787585201.119062:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f48b96c18 time=120 v=5 (1 1 0 0) 00000100:00000001:2.0:1787585201.119064:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585201.119065:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585201.119067:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1787585201.119069:0:2413:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.119072:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1787585201.119078:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-OST0000-osc-ffff940f48b94800: skip recheck: last_committed 42949696454 00000100:00000001:2.0:1787585201.119080:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1787585201.119081:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.119086:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119093:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.119095:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585201.119096:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585201.119100:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585201.119105:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500439296:192.168.203.132@tcp:400:kworker.0 00000400:00000001:2.0:1787585201.119111:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585201.119113:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585201.119114:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f7358d210. 00000400:00000200:2.0:1787585201.119116:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488deee0 00000400:00000010:2.0:1787585201.119118:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488deee0. 00000100:00000001:2.0:1787585201.119120:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585201.119123:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585201.119131:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119137:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585201.119139:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.119140:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585201.119142:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585201.119143:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585201.119145:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.119147:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585201.119151:0:2413:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482dc460 time=23 v=60 (60 60 60 60) 00000100:00001000:2.0:1787585201.119154:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482dc418 time=46 v=5 (1 1 1 1) 00000100:00000001:2.0:1787585201.119156:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585201.119163:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585201.119164:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:2.0:1787585201.119169:0:2413:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1787585201.119176:0:2413:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.119178:0:2413:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1787585201.119180:0:2413:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff940f48b94800: skip recheck: last_committed 21474836545 00000100:00000001:2.0:1787585201.119181:0:2413:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1787585201.119183:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.119186:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119192:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.119194:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585201.119195:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585201.119199:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585201.119206:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2413:1874405500439168:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585201.119210:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.119213:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585201.119226:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0ce00 x1874405500439040/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119232:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585201.119235:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585201.119238:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f440f9600. 02000000:00000001:2.0:1787585201.119242:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585201.119245:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585201.119247:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f482d8000 refcount=8 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000020:00000001:2.0:1787585201.119249:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585201.119251:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9a00. 02000000:00000001:2.0:1787585201.119255:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585201.119257:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585201.119264:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0ce00. 00000100:00000001:2.0:1787585201.119268:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585201.119269:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.119271:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585201.119275:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0dc00 x1874405500439296/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119281:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585201.119283:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585201.119285:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f440f9e00. 02000000:00000001:2.0:1787585201.119287:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585201.119288:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585201.119289:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f48b96800 refcount=3 obd=lustre-OST0000-osc-ffff940f48b94800 00000020:00000001:2.0:1787585201.119291:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585201.119293:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9f00. 02000000:00000001:2.0:1787585201.119294:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585201.119295:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585201.119297:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0dc00. 00000100:00000001:2.0:1787585201.119299:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585201.119300:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.119302:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585201.119305:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0f800 x1874405500439168/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585286 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.119311:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585201.119312:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585201.119313:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f440f9a00. 02000000:00000001:2.0:1787585201.119315:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585201.119316:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585201.119317:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f482dc000 refcount=3 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000020:00000001:2.0:1787585201.119319:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585201.119321:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9b00. 02000000:00000001:2.0:1787585201.119322:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585201.119323:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585201.119325:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0f800. 00000100:00000001:2.0:1787585201.119326:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585201.119327:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.119330:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.119336:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.119338:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787585201.119340:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.119342:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:3.0:1787585201.121982:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.121999:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585201.122004:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.122006:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.122014:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.122022:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585201.122038:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.122043:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.122048:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.122053:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 25 MB=0x6a8c30a04c980 00000400:00000200:3.0:1787585201.122057:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585201.122062:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e4d [1] + 224 00000800:00000001:3.0:1787585201.122070:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.122076:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585201.122079:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585201.122083:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585201.122091:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585201.122099:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585201.122109:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585201.122132:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585201.122141:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585201.122145:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.122403:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.122408:0:2413:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1787585201.122414:0:2413:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787585201.122416:0:2413:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787585201.122419:0:2413:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f7358d630. 00000400:00000200:2.0:1787585201.122424:0:2413:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f488de000 00000400:00000010:2.0:1787585201.122426:0:2413:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f488de000. 00000100:00000001:2.0:1787585201.122429:0:2413:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585201.122434:0:2413:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787585201.122443:0:2413:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.122450:0:2413:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787585201.122451:0:2413:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.122454:0:2413:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:2.0:1787585201.122457:0:2413:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1787585201.122459:0:2413:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1787585201.122461:0:2413:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787585201.122464:0:2413:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1787585201.122469:0:2413:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f60d3c460 time=23 v=5 (5 5 5 5) 00000100:00001000:2.0:1787585201.122472:0:2413:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f60d3c418 time=66 v=5 (1 1 1 1) 00000100:00000001:2.0:1787585201.122475:0:2413:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1787585201.122476:0:2413:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787585201.122479:0:2413:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1787585201.122481:0:2413:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.122484:0:2413:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787585201.122488:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.122494:0:2413:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787585201.122497:0:2413:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787585201.122499:0:2413:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787585201.122502:0:2413:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787585201.122518:0:2413:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f72b0df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2413:1874405500438912:192.168.203.132@tcp:400:kworker.0 00000100:00000001:2.0:1787585201.122523:0:2413:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.122526:0:2413:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787585201.122529:0:2413:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f72b0df80 x1874405500438912/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585217 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585201.122536:0:2413:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787585201.122538:0:2413:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787585201.122541:0:2413:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f440f9400. 02000000:00000001:2.0:1787585201.122545:0:2413:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787585201.122547:0:2413:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787585201.122549:0:2413:0:(genops.c:1223:class_import_put()) import ffff940f60d3c000 refcount=3 obd=MGC192.168.203.132@tcp 00000020:00000001:2.0:1787585201.122551:0:2413:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787585201.122553:0:2413:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940e410e9300. 02000000:00000001:2.0:1787585201.122557:0:2413:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787585201.122568:0:2413:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787585201.122571:0:2413:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f72b0df80. 00000100:00000001:2.0:1787585201.122574:0:2413:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787585201.122576:0:2413:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.122578:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787585201.122586:0:2413:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787585201.122588:0:2413:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787585201.122590:0:2413:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787585201.122592:0:2413:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585201.549869:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585201.549878:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585201.549883:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585201.549885:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787585201.549984:0:247673:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1787585201.549987:0:247673:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1787585201.549990:0:247673:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585202.573699:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585202.573708:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585202.573715:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585202.573716:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787585202.573816:0:241963:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787585202.573819:0:241963:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787585202.573822:0:241963:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585203.599371:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585203.599379:0:279548:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1787585203.599381:0:279548:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1787585203.599396:0:279548:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585203.599401:0:279548:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1787585203.599403:0:279548:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585203.599405:0:279548:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585203.599407:0:279548:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585203.599429:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585203.599430:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787585203.599467:0:235982:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787585203.599470:0:235982:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787585203.599473:0:235982:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585204.623126:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585204.623135:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585204.623143:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585204.623144:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787585204.623355:0:247674:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1787585204.623360:0:247674:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1787585204.623363:0:247674:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1787585204.941483:0:280011:0:(connection.c:75:cpu_latency_work()) work item of ffff940f44e33a20 (cpu 2) has reached its deadline 4307688661, at 4307688704 00000100:00000010:2.0:1787585204.941506:0:280011:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff940f715ba4e0. 00000100:00000040:3.0:1787585204.941574:0:279548:0:(connection.c:75:cpu_latency_work()) work item of ffff940f44e33b30 (cpu 3) has reached its deadline 4307688661, at 4307688704 00000100:00000010:3.0:1787585204.941601:0:279548:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff940f715ba240. 00010000:00000001:3.0:1787585205.645506:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585205.645514:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585205.645520:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585205.645521:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787585205.645644:0:235981:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585205.645647:0:235981:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585205.645650:0:235981:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1787585206.080117:0:284835:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221592:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787585206.221604:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1787585206.221613:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f566c1180. 00000020:00000040:0.0:1787585206.221619:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f60d3c000 refcount=4 obd=MGC192.168.203.132@tcp 00000100:00000001:0.0:1787585206.221623:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787585206.221626:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787585206.221631:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787585206.221638:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940f56ddb300. 00000100:00000001:0.0:1787585206.221644:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221656:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging df00bfb3-0c7b-4ae9-9143-8cf10122546d->MGS req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787585206.221667:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787585206.221672:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221679:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000025093829] to pc [ptlrpcd_00_00+0] req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.221700:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221703:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787585206.221707:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1787585206.221710:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f566c0700. 00000020:00000040:0.0:1787585206.221712:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f482d8000 refcount=9 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000100:00000001:0.0:1787585206.221715:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787585206.221716:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787585206.221718:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787585206.221736:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940f56ddb100. 00000100:00000001:0.0:1787585206.221740:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221745:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0000_UUID req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787585206.221752:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787585206.221755:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221758:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008bba41a6] to pc [ptlrpcd_00_01+1] req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.221766:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221769:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787585206.221771:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1787585206.221774:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f566c2300. 00000020:00000040:0.0:1787585206.221776:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f482dc000 refcount=4 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000100:00000001:0.0:1787585206.221779:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787585206.221780:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787585206.221781:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787585206.221784:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940f56ddb500. 00000100:00000001:0.0:1787585206.221787:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221791:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-MDT0001_UUID req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787585206.221798:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787585206.221800:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221804:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b10ee971] to pc [ptlrpcd_00_00+0] req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.221811:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221814:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787585206.221817:0:232188:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1787585206.221819:0:232188:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff940f566c1880. 00000020:00000040:0.0:1787585206.221821:0:232188:0:(genops.c:1210:class_import_get()) import ffff940f48b96800 refcount=4 obd=lustre-OST0000-osc-ffff940f48b94800 00000100:00000001:0.0:1787585206.221823:0:232188:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787585206.221829:0:232188:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787585206.221831:0:232188:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787585206.221835:0:232188:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff940f56ddb200. 00000100:00000001:0.0:1787585206.221838:0:232188:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221842:0:232188:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0000_UUID req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787585206.221848:0:232188:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787585206.221850:0:232188:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221853:0:232188:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009a25570d] to pc [ptlrpcd_00_01+1] req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.221860:0:232188:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.221862:0:232188:0:(pinger.c:229:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787585206.221866:0:232188:0:(pinger.c:247:ptlrpc_pinger_process_import()) fac6ef4a-3c6f-40fa-8410-e2886a870891->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1787585206.221872:0:232188:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (13027) 00000100:00000001:0.0:1787585206.221902:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787585206.221907:0:2411:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787585206.221909:0:2411:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787585206.221915:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.221923:0:2411:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787585206.221926:0:2411:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.221930:0:2411:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787585206.221933:0:2411:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787585206.221936:0:2411:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787585206.221939:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f566c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2411:1874405500439552:192.168.203.132@tcp:400:kworker.0 00000100:00000001:0.0:1787585206.221944:0:2411:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787585206.221982:0:2411:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787585206.221986:0:2411:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.221989:0:2411:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787585206.221995:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f60807800. 02000000:00000001:0.0:1787585206.221997:0:2411:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787585206.222015:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:0.0:1787585206.222021:0:2411:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f67707d10. 00000400:00000010:0.0:1787585206.222029:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f73302b28. 00000100:00000200:0.0:1787585206.222036:0:2411:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439552, portal 10 00000100:00000001:0.0:1787585206.222039:0:2411:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787585206.222041:0:2411:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392328181504 : -118681381370112 : ffff940f566c0700) 00000100:00000040:0.0:1787585206.222052:0:2411:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222060:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787585206.222062:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:0.0:1787585206.222066:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f733027f8. 00000100:00000200:0.0:1787585206.222070:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500439552, offset 0 00000400:00000200:0.0:1787585206.222077:0:2411:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222085:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222094:0:2411:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:0.0:1787585206.222099:0:2411:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787585206.222103:0:2411:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:0.0:1787585206.222107:0:2411:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787585206.222110:0:2411:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222114:0:2411:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538728:538728:256:4294967295] 192.168.203.132@tcp LPNI seq info [538693:538693:8:4294967295] 00000400:00000200:0.0:1787585206.222118:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222128:0:2411:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:0.0:1787585206.222134:0:2411:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:0.0:1787585206.222138:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f56ddb000. 00000800:00000200:0.0:1787585206.222145:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:0.0:1787585206.222151:0:2411:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:0.0:1787585206.222156:0:2411:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f56ddb000 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1787585206.222164:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.222168:0:2411:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: using a timeout of 22326 usecs (23 jiffies) 00000400:00000200:0.0:1787585206.222172:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:0.0:1787585206.222174:0:2411:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307690007 00000100:00000010:0.0:1787585206.222188:0:2411:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f619d8840. 00000100:00000001:0.0:1787585206.222195:0:2411:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request now active for cpu 2 00000100:00000001:0.0:1787585206.222204:0:2411:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307690007 00000100:00000010:0.0:1787585206.222207:0:2411:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff940f619d8e40. 00000100:00000001:0.0:1787585206.222210:0:2411:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff940f48b94800: PM QoS request now active for cpu 3 00000100:00000001:0.0:1787585206.222214:0:2411:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.222216:0:2411:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.222219:0:2411:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787585206.222224:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222230:0:2411:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787585206.222232:0:2411:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.222234:0:2411:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787585206.222236:0:2411:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787585206.222238:0:2411:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787585206.222241:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f566c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2411:1874405500439808:192.168.203.132@tcp:400:kworker.0 00000100:00000001:0.0:1787585206.222244:0:2411:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787585206.222247:0:2411:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787585206.222248:0:2411:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.222250:0:2411:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787585206.222253:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f60807200. 02000000:00000001:0.0:1787585206.222254:0:2411:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787585206.222258:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:0.0:1787585206.222261:0:2411:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f67707478. 00000400:00000010:0.0:1787585206.222265:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f73302660. 00000100:00000200:0.0:1787585206.222268:0:2411:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439808, portal 4 00000100:00000001:0.0:1787585206.222270:0:2411:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787585206.222271:0:2411:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392328185984 : -118681381365632 : ffff940f566c1880) 00000100:00000040:0.0:1787585206.222277:0:2411:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222283:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787585206.222290:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:0.0:1787585206.222292:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f733026e8. 00000100:00000200:0.0:1787585206.222295:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1874405500439808, offset 0 00000400:00000200:0.0:1787585206.222299:0:2411:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222303:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222307:0:2411:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:0.0:1787585206.222309:0:2411:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787585206.222312:0:2411:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:0.0:1787585206.222314:0:2411:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787585206.222317:0:2411:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222320:0:2411:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538729:538729:255:4294967295] 192.168.203.132@tcp LPNI seq info [538694:538694:7:4294967295] 00000400:00000200:0.0:1787585206.222324:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222330:0:2411:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:0.0:1787585206.222334:0:2411:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:0.0:1787585206.222337:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f56ddbe00. 00000800:00000200:0.0:1787585206.222341:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:0.0:1787585206.222345:0:2411:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:0.0:1787585206.222348:0:2411:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f56ddbe00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1787585206.222351:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.222354:0:2411:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: using a timeout of 122410 usecs (123 jiffies) 00000400:00000200:0.0:1787585206.222357:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:0.0:1787585206.222359:0:2411:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307690107 00000100:00000001:0.0:1787585206.222364:0:2411:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 2, simply extend its deadline from 4307690007 00000100:00000001:0.0:1787585206.222367:0:2411:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307690107 00000100:00000001:0.0:1787585206.222370:0:2411:0:(niobuf.c:685:kick_cpu_latency()) lustre-OST0000-osc-ffff940f48b94800: PM QoS request active for cpu 3, simply extend its deadline from 4307690007 00000100:00000001:0.0:1787585206.222373:0:2411:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.222375:0:2411:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.222377:0:2411:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.222380:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.222411:0:2411:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787585206.222419:0:2411:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1787585206.222422:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787585206.222423:0:2411:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787585206.222425:0:2411:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787585206.222428:0:2411:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00000001:0.0:1787585206.222430:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.222432:0:2411:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787585206.222434:0:2411:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.222435:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787585206.222436:0:2411:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787585206.222437:0:2411:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787585206.222442:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222450:0:2411:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787585206.222451:0:2411:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.222453:0:2411:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787585206.222456:0:2411:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787585206.222458:0:2411:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787585206.222466:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f566c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2411:1874405500439424:192.168.203.132@tcp:400:kworker.0 00000100:00000001:0.0:1787585206.222470:0:2411:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787585206.222473:0:2411:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787585206.222474:0:2411:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.222476:0:2411:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787585206.222479:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f60807600. 02000000:00000001:0.0:1787585206.222482:0:2411:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787585206.222485:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:0.0:1787585206.222488:0:2411:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f67707528. 00000400:00000010:0.0:1787585206.222492:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f73302088. 00000100:00000200:0.0:1787585206.222495:0:2411:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439424, portal 25 00000100:00000001:0.0:1787585206.222497:0:2411:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787585206.222498:0:2411:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392328184192 : -118681381367424 : ffff940f566c1180) 00000100:00000040:0.0:1787585206.222503:0:2411:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222509:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787585206.222516:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:0.0:1787585206.222518:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f73302bb0. 00000100:00000200:0.0:1787585206.222521:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1874405500439424, offset 0 00000400:00000200:0.0:1787585206.222526:0:2411:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222530:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222533:0:2411:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:0.0:1787585206.222536:0:2411:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787585206.222539:0:2411:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:0.0:1787585206.222541:0:2411:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787585206.222543:0:2411:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222547:0:2411:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538730:538730:254:4294967295] 192.168.203.132@tcp LPNI seq info [538695:538695:6:4294967295] 00000400:00000200:0.0:1787585206.222551:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222557:0:2411:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:0.0:1787585206.222561:0:2411:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:0.0:1787585206.222564:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f56ddb600. 00000800:00000200:0.0:1787585206.222568:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:0.0:1787585206.222572:0:2411:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:0.0:1787585206.222574:0:2411:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f56ddb600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1787585206.222577:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787585206.222580:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:0.0:1787585206.222582:0:2411:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 2 is 4307693781 00000100:00000001:0.0:1787585206.222586:0:2411:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4307690107 00000100:00000001:0.0:1787585206.222589:0:2411:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS new deadline estimation for cpu 3 is 4307693781 00000100:00000001:0.0:1787585206.222591:0:2411:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.203.132@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4307690107 00000100:00000001:0.0:1787585206.222594:0:2411:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.222596:0:2411:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.222598:0:2411:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787585206.222601:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222608:0:2411:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787585206.222615:0:2411:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.222618:0:2411:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787585206.222620:0:2411:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787585206.222621:0:2411:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787585206.222624:0:2411:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff940f566c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2411:1874405500439680:192.168.203.132@tcp:400:kworker.0 00000100:00000001:0.0:1787585206.222628:0:2411:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787585206.222630:0:2411:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787585206.222631:0:2411:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.222633:0:2411:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787585206.222636:0:2411:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff940f60806800. 02000000:00000001:0.0:1787585206.222638:0:2411:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787585206.222642:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000010:0.0:1787585206.222644:0:2411:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff940f67707000. 00000400:00000010:0.0:1787585206.222647:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f733022a8. 00000100:00000200:0.0:1787585206.222650:0:2411:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874405500439680, portal 10 00000100:00000001:0.0:1787585206.222652:0:2411:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787585206.222653:0:2411:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446625392328188672 : -118681381362944 : ffff940f566c2300) 00000100:00000040:0.0:1787585206.222659:0:2411:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.222665:0:2411:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787585206.222667:0:2411:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.203.132@tcp 00000400:00000010:0.0:1787585206.222670:0:2411:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff940f73302cc0. 00000100:00000200:0.0:1787585206.222673:0:2411:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874405500439680, offset 0 00000400:00000200:0.0:1787585206.222677:0:2411:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222681:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222685:0:2411:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.132@tcp local destination 00000400:00000200:0.0:1787585206.222687:0:2411:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787585206.222690:0:2411:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.32@tcp 00000400:00000200:0.0:1787585206.222692:0:2411:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.32@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787585206.222694:0:2411:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.132@tcp 00000400:00000200:0.0:1787585206.222696:0:2411:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.32@tcp NI seq info: [538731:538731:253:4294967295] 192.168.203.132@tcp LPNI seq info [538696:538696:5:4294967295] 00000400:00000200:0.0:1787585206.222700:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:0.0:1787585206.222705:0:2411:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp:) -> 192.168.203.132@tcp(192.168.203.132@tcp:192.168.203.132@tcp) : PUT try# 0 00000800:00000200:0.0:1787585206.222715:0:2411:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.132@tcp 00000800:00000010:0.0:1787585206.222719:0:2411:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff940f56ddbb00. 00000800:00000200:0.0:1787585206.222723:0:2411:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff940f6099a900] -> 12345-192.168.203.132@tcp (5) 00000800:00000200:0.0:1787585206.222727:0:2411:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.132@tcp ip 192.168.203.132:988 00000800:00000200:0.0:1787585206.222730:0:2411:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff940f56ddbb00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1787585206.222733:0:2411:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.222736:0:2411:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: using a timeout of 25400 usecs (26 jiffies) 00000400:00000200:0.0:1787585206.222739:0:2411:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:0.0:1787585206.222741:0:2411:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 2 is 4307690011 00000100:00000001:0.0:1787585206.222744:0:2411:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 2, keep current deadline 4307693781 00000100:00000001:0.0:1787585206.222746:0:2411:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS new deadline estimation for cpu 3 is 4307690011 00000100:00000001:0.0:1787585206.222748:0:2411:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff940f48b94800: PM QoS request active for cpu 3, keep current deadline 4307693781 00000100:00000001:0.0:1787585206.222750:0:2411:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.222752:0:2411:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.222754:0:2411:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.222757:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1787585206.222877:0:2405:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585206.222893:0:2405:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f56ddb000. 00000400:00000200:3.0:1787585206.222901:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585206.222906:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585206.222912:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f733027f8 00000400:00000010:3.0:1787585206.222915:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f733027f8. 00000100:00000001:3.0:1787585206.222920:0:2405:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585206.222925:0:2405:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.222935:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585206.222938:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.222946:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585206.222947:0:2405:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585206.222960:0:2405:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585206.222962:0:2405:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f56ddbe00. 00000400:00000200:3.0:1787585206.222964:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585206.222968:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585206.222971:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f733026e8 00000400:00000010:3.0:1787585206.222978:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f733026e8. 00000100:00000001:3.0:1787585206.222981:0:2405:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585206.222984:0:2405:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.222992:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585206.222995:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.223002:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585206.223004:0:2405:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585206.223011:0:2405:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585206.223013:0:2405:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f56ddb600. 00000400:00000200:3.0:1787585206.223015:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585206.223019:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585206.223021:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f73302bb0 00000400:00000010:3.0:1787585206.223023:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f73302bb0. 00000100:00000001:3.0:1787585206.223025:0:2405:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585206.223029:0:2405:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.223037:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585206.223040:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.223047:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585206.223048:0:2405:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787585206.223103:0:2405:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787585206.223105:0:2405:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff940f56ddbb00. 00000400:00000200:3.0:1787585206.223108:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585206.223111:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000400:00000200:3.0:1787585206.223114:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f73302cc0 00000400:00000010:3.0:1787585206.223116:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f73302cc0. 00000100:00000001:3.0:1787585206.223118:0:2405:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787585206.223122:0:2405:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.223129:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787585206.223137:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.223144:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787585206.223145:0:2405:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787585206.223368:0:2410:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787585206.223372:0:2410:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:2.0:1787585206.226145:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226158:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1787585206.226173:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226176:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787585206.226196:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226203:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:2.0:1787585206.226212:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226217:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226222:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226226:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 25 MB=0x6a8c30a04cb80 00000400:00000200:2.0:1787585206.226231:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226237:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e9d [1] + 224 00000800:00000001:2.0:1787585206.226244:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226248:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787585206.226252:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1787585206.226256:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:2.0:1787585206.226264:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585206.226272:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787585206.226284:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585206.226303:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1787585206.226311:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226315:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1787585206.226317:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226319:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787585206.226322:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226326:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:2.0:1787585206.226330:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226333:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226336:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226339:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04cc00 00000400:00000200:2.0:1787585206.226342:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226346:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e8d [1] + 224 00000800:00000001:2.0:1787585206.226358:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226361:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787585206.226363:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1787585206.226367:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:2.0:1787585206.226370:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585206.226375:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787585206.226427:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585206.226436:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1787585206.226441:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226445:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:2.0:1787585206.226447:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226449:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787585206.226452:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226456:0:2404:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:2.0:1787585206.226460:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226463:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226466:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226469:0:2404:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 10 MB=0x6a8c30a04cc80 00000400:00000200:2.0:1787585206.226472:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:2.0:1787585206.226476:0:2404:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.132@tcp of length 224/224 into md 0x513ea5 [1] + 224 00000800:00000001:2.0:1787585206.226480:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226485:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787585206.226488:0:2404:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1787585206.226491:0:2404:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:2.0:1787585206.226494:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787585206.226498:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787585206.226507:0:2404:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787585206.226526:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:2.0:1787585206.226531:0:2404:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:2.0:1787585206.226534:0:2404:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585206.227068:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585206.227076:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:3.0:1787585206.227078:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585206.227081:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585206.227086:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000100:00000001:0.0:1787585206.227378:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1787585206.227491:0:2405:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.32@tcp(192.168.203.32@tcp) <- 192.168.203.132@tcp : PUT - for me 00000400:00000200:3.0:1787585206.227499:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585206.227503:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585206.227506:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585206.227510:0:2405:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.132@tcp of length 224 into portal 4 MB=0x6a8c30a04cd00 00000400:00000200:3.0:1787585206.227514:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.132@tcp to cpt 1 00000400:00000200:3.0:1787585206.227519:0:2405:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.203.132@tcp of length 224/224 into md 0x513e95 [1] + 224 00000800:00000001:3.0:1787585206.227525:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585206.227532:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787585206.227535:0:2405:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787585206.227539:0:2405:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.32@tcp->192.168.203.132@tcp: PUT: OK 00000100:00000001:3.0:1787585206.227544:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787585206.227551:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787585206.227561:0:2405:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787585206.227569:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:3.0:1787585206.227574:0:2405:0:(socklnd_cb.c:281:ksocknal_receive()) Process entered 00000800:00000001:3.0:1787585206.227576:0:2405:0:(socklnd_cb.c:314:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.227607:0:2411:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1787585206.227615:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787585206.227617:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787585206.227620:0:2411:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f67707528. 00000400:00000200:0.0:1787585206.227624:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f73302088 00000400:00000010:0.0:1787585206.227626:0:2411:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f73302088. 00000100:00000001:0.0:1787585206.227629:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787585206.227635:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787585206.227644:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227650:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787585206.227652:0:2411:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.227655:0:2411:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1787585206.227668:0:2411:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1787585206.227671:0:2411:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1787585206.227673:0:2411:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.227676:0:2411:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1787585206.227684:0:2411:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f60d3c460 time=28 v=5 (5 5 5 5) 00000100:00001000:0.0:1787585206.227687:0:2411:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f60d3c418 time=71 v=5 (1 1 1 1) 00000100:00000001:0.0:1787585206.227690:0:2411:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1787585206.227691:0:2411:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1787585206.227694:0:2411:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1787585206.227696:0:2411:0:(ldlm_request.c:1657:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.227699:0:2411:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.227704:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227711:0:2411:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.227714:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787585206.227716:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787585206.227720:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787585206.227727:0:2411:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f566c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:df00bfb3-0c7b-4ae9-9143-8cf10122546d:2411:1874405500439424:192.168.203.132@tcp:400:kworker.0 00000400:00000001:0.0:1787585206.227734:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787585206.227735:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787585206.227737:0:2411:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f67707000. 00000400:00000200:0.0:1787585206.227740:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f733022a8 00000400:00000010:0.0:1787585206.227741:0:2411:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f733022a8. 00000100:00000001:0.0:1787585206.227744:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787585206.227747:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787585206.227757:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227763:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787585206.227765:0:2411:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.227767:0:2411:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1787585206.227773:0:2411:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1787585206.227775:0:2411:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1787585206.227776:0:2411:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.227779:0:2411:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1787585206.227785:0:2411:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482dc460 time=28 v=60 (60 60 60 60) 00000100:00001000:0.0:1787585206.227788:0:2411:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482dc418 time=51 v=5 (1 1 1 1) 00000100:00000001:0.0:1787585206.227791:0:2411:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1787585206.227793:0:2411:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1787585206.227795:0:2411:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1787585206.227799:0:2411:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1787585206.227807:0:2411:0:(ldlm_request.c:1672:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.227810:0:2411:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1787585206.227813:0:2411:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff940f48b94800: skip recheck: last_committed 21474836545 00000100:00000001:0.0:1787585206.227815:0:2411:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1787585206.227816:0:2411:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.227820:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227828:0:2411:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.227830:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787585206.227831:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787585206.227834:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787585206.227842:0:2411:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f566c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2411:1874405500439680:192.168.203.132@tcp:400:kworker.0 00000400:00000001:0.0:1787585206.227848:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787585206.227849:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787585206.227851:0:2411:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f67707d10. 00000400:00000200:0.0:1787585206.227854:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f73302b28 00000400:00000010:0.0:1787585206.227856:0:2411:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f73302b28. 00000100:00000001:0.0:1787585206.227858:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787585206.227862:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787585206.227892:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227900:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787585206.227902:0:2411:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.227904:0:2411:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1787585206.227906:0:2411:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1787585206.227907:0:2411:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1787585206.227909:0:2411:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.227910:0:2411:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1787585206.227915:0:2411:0:(import.c:2149:obd_at_measure()) add 60 to ffff940f482d8460 time=28 v=60 (60 60 60 60) 00000100:00001000:0.0:1787585206.227918:0:2411:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f482d8418 time=71 v=5 (1 1 1 1) 00000100:00000001:0.0:1787585206.227920:0:2411:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1787585206.227922:0:2411:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1787585206.227923:0:2411:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1787585206.227926:0:2411:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.227929:0:2411:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1787585206.227930:0:2411:0:(client.c:3121:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff940f48b94800: skip recheck: last_committed 30064771622 00000100:00000001:0.0:1787585206.227932:0:2411:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1787585206.227933:0:2411:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.227938:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227945:0:2411:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.227946:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787585206.227948:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787585206.227951:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787585206.227959:0:2411:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f566c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2411:1874405500439552:192.168.203.132@tcp:400:kworker.0 00000400:00000001:0.0:1787585206.227965:0:2411:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787585206.227966:0:2411:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787585206.227968:0:2411:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff940f67707478. 00000400:00000200:0.0:1787585206.227976:0:2411:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff940f73302660 00000400:00000010:0.0:1787585206.227978:0:2411:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff940f73302660. 00000100:00000001:0.0:1787585206.227980:0:2411:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787585206.227984:0:2411:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787585206.227992:0:2411:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.227999:0:2411:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787585206.228001:0:2411:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.228003:0:2411:0:(client.c:1593:after_reply()) Process entered 02000000:00000001:0.0:1787585206.228004:0:2411:0:(sec.c:1125:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1787585206.228006:0:2411:0:(pack_generic.c:605:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1787585206.228007:0:2411:0:(pack_generic.c:623:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787585206.228009:0:2411:0:(sec.c:1181:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1787585206.228014:0:2411:0:(import.c:2149:obd_at_measure()) add 5 to ffff940f48b96c60 time=28 v=5 (5 5 5 5) 00000100:00001000:0.0:1787585206.228017:0:2411:0:(import.c:2149:obd_at_measure()) add 1 to ffff940f48b96c18 time=125 v=5 (1 1 0 0) 00000100:00000001:0.0:1787585206.228019:0:2411:0:(client.c:1495:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1787585206.228020:0:2411:0:(client.c:1533:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1787585206.228022:0:2411:0:(ldlm_request.c:1653:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1787585206.228024:0:2411:0:(ldlm_request.c:1683:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.228027:0:2411:0:(client.c:3109:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1787585206.228028:0:2411:0:(client.c:3121:ptlrpc_free_committed()) lustre-OST0000-osc-ffff940f48b94800: skip recheck: last_committed 42949696454 00000100:00000001:0.0:1787585206.228030:0:2411:0:(client.c:3122:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1787585206.228032:0:2411:0:(client.c:1797:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787585206.228036:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.228043:0:2411:0:(client.c:2354:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787585206.228045:0:2411:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787585206.228046:0:2411:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787585206.228050:0:2411:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787585206.228058:0:2411:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff940f566c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:fac6ef4a-3c6f-40fa-8410-e2886a870891:2411:1874405500439808:192.168.203.132@tcp:400:kworker.0 00000100:00000001:0.0:1787585206.228069:0:2411:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.228072:0:2411:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787585206.228076:0:2411:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f566c1180 x1874405500439424/t0(0) o400->MGC192.168.203.132@tcp@192.168.203.132@tcp:26/25 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.228083:0:2411:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787585206.228086:0:2411:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787585206.229544:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f60807600. 02000000:00000001:0.0:1787585206.229551:0:2411:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787585206.229555:0:2411:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787585206.229557:0:2411:0:(genops.c:1223:class_import_put()) import ffff940f60d3c000 refcount=3 obd=MGC192.168.203.132@tcp 00000020:00000001:0.0:1787585206.229560:0:2411:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787585206.229563:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940f56ddb300. 02000000:00000001:0.0:1787585206.229567:0:2411:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787585206.229569:0:2411:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787585206.229571:0:2411:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f566c1180. 00000100:00000001:0.0:1787585206.229574:0:2411:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787585206.229576:0:2411:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.229580:0:2411:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787585206.229590:0:2411:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f566c2300 x1874405500439680/t0(0) o400->lustre-MDT0001-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.229599:0:2411:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787585206.229601:0:2411:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787585206.229602:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f60806800. 02000000:00000001:0.0:1787585206.229606:0:2411:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787585206.229607:0:2411:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787585206.229608:0:2411:0:(genops.c:1223:class_import_put()) import ffff940f482dc000 refcount=3 obd=lustre-MDT0001-mdc-ffff940f48b94800 00000020:00000001:0.0:1787585206.229611:0:2411:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787585206.229612:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940f56ddb500. 02000000:00000001:0.0:1787585206.229614:0:2411:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787585206.229616:0:2411:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787585206.229617:0:2411:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f566c2300. 00000100:00000001:0.0:1787585206.229619:0:2411:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787585206.229620:0:2411:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.229622:0:2411:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787585206.229626:0:2411:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f566c0700 x1874405500439552/t0(0) o400->lustre-MDT0000-mdc-ffff940f48b94800@192.168.203.132@tcp:12/10 lens 224/224 e 0 to 0 dl 1787585291 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.229642:0:2411:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787585206.229644:0:2411:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787585206.229645:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f60807800. 02000000:00000001:0.0:1787585206.229648:0:2411:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787585206.229650:0:2411:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787585206.229651:0:2411:0:(genops.c:1223:class_import_put()) import ffff940f482d8000 refcount=8 obd=lustre-MDT0000-mdc-ffff940f48b94800 00000020:00000001:0.0:1787585206.229652:0:2411:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787585206.229654:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940f56ddb100. 02000000:00000001:0.0:1787585206.229656:0:2411:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787585206.229657:0:2411:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787585206.229658:0:2411:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f566c0700. 00000100:00000001:0.0:1787585206.229660:0:2411:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787585206.229661:0:2411:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.229663:0:2411:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787585206.229667:0:2411:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff940f566c1880 x1874405500439808/t0(0) o400->lustre-OST0000-osc-ffff940f48b94800@192.168.203.132@tcp:28/4 lens 224/224 e 0 to 0 dl 1787585222 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787585206.229673:0:2411:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787585206.229674:0:2411:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787585206.229676:0:2411:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff940f60807200. 02000000:00000001:0.0:1787585206.229678:0:2411:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787585206.229690:0:2411:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787585206.229692:0:2411:0:(genops.c:1223:class_import_put()) import ffff940f48b96800 refcount=3 obd=lustre-OST0000-osc-ffff940f48b94800 00000020:00000001:0.0:1787585206.229694:0:2411:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787585206.229696:0:2411:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff940f56ddb200. 02000000:00000001:0.0:1787585206.229698:0:2411:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787585206.229699:0:2411:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787585206.229700:0:2411:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff940f566c1880. 00000100:00000001:0.0:1787585206.229702:0:2411:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787585206.229704:0:2411:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.229706:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787585206.229714:0:2411:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787585206.229716:0:2411:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787585206.229720:0:2411:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787585206.229722:0:2411:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1787585206.429054:0:284871:0:(debug.c:652:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:1.0:1787585206.429061:0:284871:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity test_816: @@@@@@ FAIL: client: import is not in IDLE state after 50 00000001:00000001:1.0:1787585206.434727:0:284871:0:(debug.c:655:libcfs_debug_mark_buffer()) ************************************************** 00010000:00000001:3.0:1787585206.669715:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585206.669723:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585206.669727:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585206.669729:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787585206.670342:0:247673:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1787585206.670458:0:247673:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1787585206.670462:0:247673:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1787585207.693443:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585207.693451:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787585207.693456:0:279548:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787585207.693457:0:279548:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787585207.693522:0:241963:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787585207.693525:0:241963:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787585207.693528:0:241963:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:1.0:1787585207.777644:0:284972:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) Debug log: 2596 lines, 2596 kept, 0 dropped, 0 bad.