00000100:00000010:1.0F:1788971833.288877:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80.
02000000:00000010:1.0:1788971833.288905:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6a00.
00000100:00000010:1.0:1788971833.288947:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800.
02000000:00000010:1.0:1788971833.288953:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6c00.
02000000:00000010:1.0:1788971833.288987:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c200.
00000400:00000200:1.0:1788971833.288996:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971833.289000:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971833.289006:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971833.289013:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065728, portal 10
00000400:00000010:1.0:1788971833.289021:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330.
00000100:00000200:1.0:1788971833.289024:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179065728, offset 0
00000400:00000200:1.0:1788971833.289033:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971833.289052:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971833.289059:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971833.289064:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971833.289068:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971833.289073:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971833.289075:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971833.289079:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [299:299:256:4294967295] 192.168.203.124@tcp LPNI seq info [299:299:8:4294967295]
00000400:00000200:1.0:1788971833.289083:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971833.289093:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971833.289099:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971833.289104:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb496ee000.
00000800:00000200:1.0:1788971833.289111:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971833.289118:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971833.289123:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb496ee000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971833.289148:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971833.289153:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9ff00.
00000100:00000010:1.0:1788971833.289162:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f840.
02000000:00000010:1.0:1788971833.289180:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389de00.
00000400:00000200:1.0:1788971833.289185:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971833.289187:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78.
00000400:00000010:1.0:1788971833.289196:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445885d8.
00000100:00000200:1.0:1788971833.289198:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065600, portal 25
00000400:00000010:1.0:1788971833.289202:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588dd0.
00000100:00000200:1.0:1788971833.289204:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179065600, offset 0
00000400:00000200:1.0:1788971833.289210:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971833.289213:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971833.289216:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971833.289218:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971833.289221:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971833.289223:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971833.289225:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971833.289228:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [300:300:255:4294967295] 192.168.203.124@tcp LPNI seq info [300:300:7:4294967295]
00000400:00000200:1.0:1788971833.289231:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971833.289237:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971833.289251:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971833.289255:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637500.
00000800:00000200:1.0:1788971833.289259:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971833.289264:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971833.289266:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971833.289270:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0F:1788971833.289431:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971833.289441:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb496ee000.
00000400:00000200:2.0:1788971833.289448:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971833.289453:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971833.289458:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330
00000400:00000010:2.0:1788971833.289460:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330.
00000100:00000200:2.0:1788971833.289470:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb7800 x1875872179065728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971833.289537:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971833.289540:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637500.
00000400:00000200:2.0:1788971833.289545:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971833.289548:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971833.289551:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588dd0
00000400:00000010:2.0:1788971833.289553:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588dd0.
00000100:00000200:2.0:1788971833.289558:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb6d80 x1875872179065600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971833.293225:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293235:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971833.293243:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293248:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293253:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293256:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09380
00000400:00000200:2.0:1788971833.293261:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293267:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1055 [1] + 224
00000400:00000200:2.0:1788971833.293276:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971833.293281:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971833.293293:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb7800 x1875872179065728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971849 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971833.293354:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293358:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971833.293362:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293372:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293376:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293378:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09300
00000400:00000200:2.0:1788971833.293382:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971833.293387:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x105d [1] + 224
00000400:00000200:2.0:1788971833.293396:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971833.293400:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971833.293406:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb6d80 x1875872179065600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971849 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971833.293674:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78.
00000400:00000200:1.0:1788971833.293680:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445885d8
00000400:00000010:1.0:1788971833.293682:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445885d8.
00000100:00000200:1.0:1788971833.293690:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb6d80 x1875872179065600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971849 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971833.293700:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb6d80 x1875872179065600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971849 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971833.293721:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971833.293724:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:1.0:1788971833.293726:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971833.293731:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb7800 x1875872179065728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971849 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971833.293745:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb7800 x1875872179065728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971849 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971833.293764:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389de00.
02000000:00000010:1.0:1788971833.293770:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6a00.
00000100:00000010:1.0:1788971833.293776:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80.
02000000:00000010:1.0:1788971833.293779:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c200.
02000000:00000010:1.0:1788971833.293782:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6c00.
00000100:00000010:1.0:1788971833.293784:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800.
00000100:00000010:3.0F:1788971833.991373:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f840.
00000100:00000010:2.0:1788971833.995451:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9ff00.
00000100:00000010:2.0:1788971838.407621:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410700.
02000000:00000010:2.0:1788971838.407655:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e900.
00000100:00000010:2.0:1788971838.407710:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413b80.
02000000:00000010:2.0:1788971838.407716:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e600.
02000000:00000010:2.0:1788971838.407774:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51151000.
00000400:00000200:2.0:1788971838.407785:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971838.407790:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3210.
00000400:00000010:2.0:1788971838.407799:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8.
00000100:00000200:2.0:1788971838.407808:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065984, portal 10
00000400:00000010:2.0:1788971838.407818:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971838.407820:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179065984, offset 0
00000400:00000200:2.0:1788971838.407833:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971838.407844:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.407854:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971838.407861:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971838.407866:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971838.407871:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971838.407874:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971838.407879:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [301:301:256:4294967295] 192.168.203.124@tcp LPNI seq info [301:301:8:4294967295]
00000400:00000200:2.0:1788971838.407884:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.407894:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971838.407919:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971838.407924:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ed00.
00000800:00000200:2.0:1788971838.407932:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971838.407941:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971838.407946:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ed00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971838.407966:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971838.407972:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0.
00000100:00000010:2.0:1788971838.407984:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0.
02000000:00000010:2.0:1788971838.408013:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51150a00.
00000400:00000200:2.0:1788971838.408018:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971838.408021:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3ec8.
00000400:00000010:2.0:1788971838.408025:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0110.
00000100:00000200:2.0:1788971838.408028:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065856, portal 25
00000400:00000010:2.0:1788971838.408031:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0770.
00000100:00000200:2.0:1788971838.408034:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179065856, offset 0
00000400:00000200:2.0:1788971838.408041:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971838.408044:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.408048:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971838.408051:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971838.408054:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971838.408056:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971838.408058:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971838.408062:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [302:302:255:4294967295] 192.168.203.124@tcp LPNI seq info [302:302:7:4294967295]
00000400:00000200:2.0:1788971838.408065:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.408072:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971838.408076:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971838.408079:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00.
00000800:00000200:2.0:1788971838.408083:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971838.408088:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971838.408090:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971838.408101:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971838.408162:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971838.408167:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ed00.
00000400:00000200:2.0:1788971838.408173:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971838.408179:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971838.408186:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330
00000400:00000010:2.0:1788971838.408189:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971838.408199:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43413b80 x1875872179065984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971838.408360:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971838.408363:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00.
00000400:00000200:2.0:1788971838.408366:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971838.408370:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971838.408373:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0770
00000400:00000010:2.0:1788971838.408375:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0770.
00000100:00000200:2.0:1788971838.408381:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410700 x1875872179065856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971854 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971838.410525:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.410539:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971838.410559:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.410566:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.410573:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.410577:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09400
00000400:00000200:2.0:1788971838.410584:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.410592:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1075 [1] + 224
00000400:00000200:2.0:1788971838.410609:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971838.410615:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971838.410634:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410700 x1875872179065856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971854 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971838.410765:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3ec8.
00000400:00000200:3.0:1788971838.410772:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0110
00000400:00000010:3.0:1788971838.410776:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0110.
00000100:00000200:3.0:1788971838.410784:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410700 x1875872179065856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971854 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971838.410795:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410700 x1875872179065856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971854 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971838.410835:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51150a00.
02000000:00000010:3.0:1788971838.410844:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e900.
00000100:00000010:3.0:1788971838.410850:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410700.
00000400:00000200:2.0:1788971838.413981:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.413994:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971838.414024:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.414031:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.414038:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.414043:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09480
00000400:00000200:2.0:1788971838.414050:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971838.414057:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1065 [1] + 224
00000400:00000200:2.0:1788971838.414090:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971838.414096:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971838.414122:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43413b80 x1875872179065984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971854 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971838.422370:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3210.
00000400:00000200:3.0:1788971838.422389:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8
00000400:00000010:3.0:1788971838.422392:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8.
00000100:00000200:3.0:1788971838.422411:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43413b80 x1875872179065984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971854 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971838.422424:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43413b80 x1875872179065984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971854 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971838.422468:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51151000.
02000000:00000010:3.0:1788971838.422477:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e600.
00000100:00000010:3.0:1788971838.422484:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413b80.
00000100:00000010:3.0:1788971839.115893:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0.
00000100:00000010:2.0:1788971839.129364:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0.
00000100:00000010:2.0:1788971843.531092:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413100.
02000000:00000010:2.0:1788971843.531128:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ef00.
00000100:00000010:2.0:1788971843.531181:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413800.
02000000:00000010:2.0:1788971843.531187:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ed00.
02000000:00000010:3.0:1788971843.531356:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96400.
00000400:00000200:3.0:1788971843.531369:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971843.531375:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78b00.
00000400:00000010:3.0:1788971843.531384:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480817f8.
00000100:00000200:3.0:1788971843.531393:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066112, portal 25
00000400:00000010:3.0:1788971843.531399:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8.
00000100:00000200:3.0:1788971843.531402:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179066112, offset 0
00000400:00000200:3.0:1788971843.531424:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971843.531435:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971843.531444:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971843.531468:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971843.531474:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971843.531478:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971843.531482:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971843.531486:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [303:303:256:4294967295] 192.168.203.124@tcp LPNI seq info [303:303:8:4294967295]
00000400:00000200:3.0:1788971843.531491:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971843.531503:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971843.531510:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971843.531516:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d700.
00000800:00000200:3.0:1788971843.531524:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971843.531533:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971843.531550:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971843.531570:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971843.531576:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a4e0.
00000100:00000010:3.0:1788971843.531597:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a540.
02000000:00000010:3.0:1788971843.531623:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96e00.
00000400:00000200:3.0:1788971843.531628:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971843.531631:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78e18.
00000400:00000010:3.0:1788971843.531634:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0.
00000100:00000200:3.0:1788971843.531637:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066240, portal 10
00000400:00000010:3.0:1788971843.531642:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081770.
00000100:00000200:3.0:1788971843.531644:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179066240, offset 0
00000400:00000200:3.0:1788971843.531649:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971843.531652:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971843.531656:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971843.531658:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971843.531661:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971843.531663:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971843.531665:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971843.531667:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [304:304:255:4294967295] 192.168.203.124@tcp LPNI seq info [304:304:7:4294967295]
00000400:00000200:3.0:1788971843.531671:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971843.531695:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971843.531700:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971843.531704:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00.
00000800:00000200:3.0:1788971843.531708:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971843.531712:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971843.531723:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971843.531730:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971843.532110:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971843.532117:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d700.
00000400:00000200:2.0:1788971843.532127:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971843.532135:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971843.532142:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8
00000400:00000010:2.0:1788971843.532145:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8.
00000100:00000200:2.0:1788971843.532156:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43413100 x1875872179066112/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971843.532256:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971843.532259:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00.
00000400:00000200:2.0:1788971843.532263:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971843.532266:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971843.532269:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081770
00000400:00000010:2.0:1788971843.532270:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081770.
00000100:00000200:2.0:1788971843.532275:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43413800 x1875872179066240/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971859 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971843.534520:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.534531:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971843.534540:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.534545:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.534551:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.534556:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09500
00000400:00000200:2.0:1788971843.534561:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.534568:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1085 [1] + 224
00000400:00000200:2.0:1788971843.534584:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971843.534590:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971843.534605:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43413100 x1875872179066112/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971859 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971843.534804:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78b00.
00000400:00000200:3.0:1788971843.534810:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480817f8
00000400:00000010:3.0:1788971843.534813:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480817f8.
00000100:00000200:3.0:1788971843.534821:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43413100 x1875872179066112/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971859 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971843.534832:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43413100 x1875872179066112/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971859 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971843.534866:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96400.
02000000:00000010:3.0:1788971843.534873:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ef00.
00000100:00000010:3.0:1788971843.534879:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413100.
00000400:00000200:2.0:1788971843.536203:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.536213:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971843.536220:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.536225:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.536230:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.536234:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09580
00000400:00000200:2.0:1788971843.536239:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971843.536245:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1095 [1] + 224
00000400:00000200:2.0:1788971843.536258:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971843.536262:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971843.536274:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43413800 x1875872179066240/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971859 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971843.536439:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78e18.
00000400:00000200:3.0:1788971843.536445:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0
00000400:00000010:3.0:1788971843.536448:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0.
00000100:00000200:3.0:1788971843.536455:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43413800 x1875872179066240/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971859 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971843.536465:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43413800 x1875872179066240/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971859 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971843.536494:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96e00.
02000000:00000010:3.0:1788971843.536501:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ed00.
00000100:00000010:3.0:1788971843.536505:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413800.
00000100:00000010:2.0:1788971844.235272:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a4e0.
00000100:00000010:3.0:1788971844.239148:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a540.
00000001:02000400:2.0:1788971846.813684:0:110848:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 300: packaged LNet UAPI headers can be compiled ========================================================== 12:37:24 (1788971844)
00000020:00000010:2.0:1788971847.413591:0:110874:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166e90.
00000020:00000010:2.0:1788971847.413877:0:110874:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166e90.
00000020:00000010:3.0:1788971847.494906:0:110879:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971847.494999:0:110879:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:0.0F:1788971848.535712:0:110883:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971848.535824:0:110883:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000100:00000010:2.0:1788971848.647600:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971848.647629:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e900.
02000000:00000010:3.0:1788971848.647733:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96000.
00000400:00000200:3.0:1788971848.647745:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971848.647750:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78e18.
00000400:00000010:3.0:1788971848.647758:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0.
00000100:00000200:3.0:1788971848.647766:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066368, portal 25
00000400:00000010:3.0:1788971848.647771:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480817f8.
00000100:00000200:3.0:1788971848.647774:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179066368, offset 0
00000400:00000200:3.0:1788971848.647785:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971848.647795:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.647805:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971848.647812:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971848.647816:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971848.647820:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971848.647823:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971848.647827:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [305:305:256:4294967295] 192.168.203.124@tcp LPNI seq info [305:305:8:4294967295]
00000400:00000200:3.0:1788971848.647831:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.647840:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971848.647847:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971848.647853:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00.
00000800:00000200:3.0:1788971848.647861:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971848.647869:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971848.647875:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971848.647896:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971848.647968:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a120.
00000100:00000010:3.0:1788971848.647982:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a540.
00000100:00000010:2.0:1788971848.648014:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412a00.
02000000:00000010:2.0:1788971848.648021:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e300.
02000000:00000010:3.0:1788971848.652527:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96c00.
00000400:00000200:3.0:1788971848.652547:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971848.652553:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78b00.
00000400:00000010:3.0:1788971848.652562:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0.
00000100:00000200:3.0:1788971848.652571:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066496, portal 10
00000400:00000010:3.0:1788971848.652600:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880.
00000100:00000200:3.0:1788971848.652604:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179066496, offset 0
00000400:00000200:3.0:1788971848.652615:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971848.652626:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.652634:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971848.652640:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971848.652645:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971848.652650:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971848.652653:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971848.652657:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [306:306:255:4294967295] 192.168.203.124@tcp LPNI seq info [306:306:7:4294967295]
00000400:00000200:3.0:1788971848.652662:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.652673:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971848.652694:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971848.652701:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d400.
00000800:00000200:3.0:1788971848.652709:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971848.652719:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971848.652724:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971848.652733:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971848.652815:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971848.652820:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00.
00000400:00000200:3.0:1788971848.652827:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971848.652832:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971848.652840:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480817f8
00000400:00000010:3.0:1788971848.652842:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480817f8.
00000100:00000200:3.0:1788971848.652855:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179066368/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971864 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971848.652954:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971848.652957:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d400.
00000400:00000200:3.0:1788971848.652960:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971848.652963:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971848.652966:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880
00000400:00000010:3.0:1788971848.652975:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880.
00000100:00000200:3.0:1788971848.652982:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412a00 x1875872179066496/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971864 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971848.656766:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971848.656778:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971848.656788:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971848.656794:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971848.656800:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971848.656805:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09680
00000400:00000200:2.0:1788971848.656812:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971848.656819:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x10b5 [1] + 224
00000400:00000200:2.0:1788971848.656836:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971848.656840:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971848.656859:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412a00 x1875872179066496/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971864 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971848.657022:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78b00.
00000400:00000200:3.0:1788971848.657029:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0
00000400:00000010:3.0:1788971848.657032:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0.
00000100:00000200:3.0:1788971848.657091:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412a00 x1875872179066496/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971864 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971848.657102:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412a00 x1875872179066496/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971864 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971848.657143:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96c00.
02000000:00000010:3.0:1788971848.657150:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e300.
00000100:00000010:3.0:1788971848.657155:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412a00.
00000400:00000200:3.0:1788971848.658292:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.658596:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971848.658609:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.658614:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.658619:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.658625:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09600
00000400:00000200:3.0:1788971848.658630:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971848.658637:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x10a5 [1] + 224
00000400:00000200:3.0:1788971848.658653:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971848.658658:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971848.658672:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179066368/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971864 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971848.658861:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78e18.
00000400:00000200:2.0:1788971848.658868:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0
00000400:00000010:2.0:1788971848.658872:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0.
00000100:00000200:2.0:1788971848.658879:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179066368/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971864 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971848.658889:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179066368/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971864 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971848.658917:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96000.
02000000:00000010:2.0:1788971848.658924:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e900.
00000100:00000010:2.0:1788971848.658929:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
00000100:00000010:2.0:1788971849.351477:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a120.
00000100:00000010:3.0:1788971849.364497:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a540.
00000020:00000010:2.0:1788971849.592091:0:110887:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166e90.
00000020:00000010:2.0:1788971849.594820:0:110887:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166e90.
00000020:00000010:3.0:1788971850.652629:0:110891:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971850.652762:0:110891:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:0.0:1788971851.734799:0:110895:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971851.735001:0:110895:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000020:00000010:1.0:1788971852.775796:0:110900:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971852.775887:0:110900:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:3.0:1788971853.767440:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00.
02000000:00000010:3.0:1788971853.767475:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400.
00000100:00000010:3.0:1788971853.767512:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00.
02000000:00000010:3.0:1788971853.767518:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00.
02000000:00000010:2.0:1788971853.767541:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548200.
02000000:00000010:3.0:1788971853.767545:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97e00.
00000400:00000200:2.0:1788971853.767553:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.767554:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971853.767561:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78b00.
00000400:00000010:2.0:1788971853.767561:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3b58.
00000400:00000010:3.0:1788971853.767571:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0.
00000400:00000010:2.0:1788971853.767571:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0770.
00000100:00000200:3.0:1788971853.767579:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066752, portal 10
00000100:00000200:2.0:1788971853.767579:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066624, portal 25
00000400:00000010:2.0:1788971853.767587:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330.
00000400:00000010:3.0:1788971853.767590:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880.
00000100:00000200:2.0:1788971853.767590:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179066624, offset 0
00000100:00000200:3.0:1788971853.767596:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179066752, offset 0
00000400:00000200:2.0:1788971853.767603:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971853.767612:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.767616:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971853.767623:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971853.767628:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971853.767631:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971853.767635:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971853.767638:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971853.767642:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [307:307:256:4294967295] 192.168.203.124@tcp LPNI seq info [307:307:8:4294967295]
00000400:00000200:2.0:1788971853.767704:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971853.767714:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971853.767720:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971853.767726:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e900.
00000800:00000200:2.0:1788971853.767733:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971853.767742:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971853.767747:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971853.767759:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971853.767764:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7120.
00000100:00000010:2.0:1788971853.767773:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f71e0.
00000400:00000200:3.0:1788971853.767848:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.767854:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971853.767857:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971853.767860:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971853.767863:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971853.767865:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971853.767868:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [308:308:255:4294967295] 192.168.203.124@tcp LPNI seq info [308:308:7:4294967295]
00000400:00000200:3.0:1788971853.767872:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.767879:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971853.767883:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971853.767887:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00.
00000800:00000200:3.0:1788971853.767892:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971853.767897:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971853.767900:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971853.767906:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971853.767914:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971853.767919:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e900.
00000400:00000200:2.0:1788971853.767924:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971853.767931:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971853.767936:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330
00000400:00000010:2.0:1788971853.767939:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971853.767953:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c4e00 x1875872179066624/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971869 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971853.768073:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971853.768076:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00.
00000400:00000200:2.0:1788971853.768080:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971853.768084:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971853.768087:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880
00000400:00000010:2.0:1788971853.768089:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880.
00000100:00000200:2.0:1788971853.768094:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c6a00 x1875872179066752/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971869 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971853.775075:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775082:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971853.775089:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775093:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775097:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775114:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09700
00000400:00000200:3.0:1788971853.775120:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775127:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x10cd [1] + 224
00000400:00000200:3.0:1788971853.775139:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971853.775144:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971853.775158:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c4e00 x1875872179066624/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971869 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971853.775199:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775203:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971853.775207:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775210:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775213:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775216:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09780
00000400:00000200:3.0:1788971853.775219:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971853.775223:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x10c5 [1] + 224
00000400:00000200:3.0:1788971853.775230:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971853.775233:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971853.775239:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c6a00 x1875872179066752/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971869 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971853.775276:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78b00.
00000400:00000200:3.0:1788971853.775281:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0
00000400:00000010:3.0:1788971853.775284:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0.
00000100:00000200:3.0:1788971853.775289:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c6a00 x1875872179066752/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971869 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971853.775324:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c6a00 x1875872179066752/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971869 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971853.775357:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97e00.
02000000:00000010:3.0:1788971853.775363:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00.
00000100:00000010:3.0:1788971853.775368:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00.
00000400:00000010:2.0:1788971853.775793:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3b58.
00000400:00000200:2.0:1788971853.775798:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0770
00000400:00000010:2.0:1788971853.775800:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0770.
00000100:00000200:2.0:1788971853.775807:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c4e00 x1875872179066624/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971869 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971853.775815:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c4e00 x1875872179066624/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971869 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971853.775828:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548200.
02000000:00000010:2.0:1788971853.775832:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400.
00000100:00000010:2.0:1788971853.775835:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00.
00000020:00000010:1.0:1788971853.813931:0:110904:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971853.814083:0:110904:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:3.0:1788971854.471421:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f71e0.
00000100:00000010:2.0:1788971854.472249:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7120.
00000020:00000010:1.0:1788971854.839742:0:110908:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971854.839833:0:110908:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:2.0:1788971855.866089:0:110912:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166e90.
00000020:00000010:2.0:1788971855.866158:0:110912:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166e90.
00000020:00000010:2.0:1788971856.903536:0:110916:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166e90.
00000020:00000010:1.0:1788971856.913238:0:110916:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166e90.
00000020:00000010:0.0:1788971857.952208:0:110920:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971857.952387:0:110920:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000100:00000010:0.0:1788971858.888047:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00.
02000000:00000010:0.0:1788971858.888065:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00.
00000100:00000010:0.0:1788971858.888102:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680.
02000000:00000010:0.0:1788971858.888107:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
02000000:00000010:1.0:1788971858.888179:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2e00.
00000400:00000200:1.0:1788971858.888189:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971858.888193:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971858.888199:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971858.888205:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179066880, portal 25
00000400:00000010:1.0:1788971858.888211:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445885d8.
00000100:00000200:1.0:1788971858.888215:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179066880, offset 0
00000400:00000200:1.0:1788971858.888223:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971858.888231:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971858.888238:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971858.888243:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971858.888248:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971858.888252:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971858.888255:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971858.888258:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [309:309:256:4294967295] 192.168.203.124@tcp LPNI seq info [309:309:8:4294967295]
00000400:00000200:1.0:1788971858.888271:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971858.888281:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971858.888287:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971858.888292:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637300.
00000800:00000200:1.0:1788971858.888339:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971858.888347:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971858.888352:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971858.888364:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971858.888369:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fc00.
00000100:00000010:1.0:1788971858.888378:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9ff00.
02000000:00000010:1.0:1788971858.888399:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2000.
00000400:00000200:1.0:1788971858.888404:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971858.888408:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78.
00000400:00000010:1.0:1788971858.888412:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588088.
00000100:00000200:1.0:1788971858.888415:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067008, portal 10
00000400:00000010:1.0:1788971858.888420:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445883b8.
00000100:00000200:1.0:1788971858.888422:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179067008, offset 0
00000400:00000200:1.0:1788971858.888427:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971858.888431:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971858.888435:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971858.888439:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971858.888442:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971858.888444:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971858.888446:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971858.888449:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [310:310:255:4294967295] 192.168.203.124@tcp LPNI seq info [310:310:7:4294967295]
00000400:00000200:1.0:1788971858.888453:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971858.888459:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971858.888464:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971858.888468:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637100.
00000800:00000200:1.0:1788971858.888471:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971858.888482:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971858.888484:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971858.888491:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971858.888495:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971858.888500:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637300.
00000400:00000200:2.0:1788971858.888506:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971858.888512:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971858.888518:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445885d8
00000400:00000010:2.0:1788971858.888521:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445885d8.
00000100:00000200:2.0:1788971858.888529:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353ce00 x1875872179066880/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971874 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971858.888559:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971858.888561:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637100.
00000400:00000200:2.0:1788971858.888564:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971858.888567:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971858.888570:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445883b8
00000400:00000010:2.0:1788971858.888572:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445883b8.
00000100:00000200:2.0:1788971858.888577:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353e680 x1875872179067008/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971874 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971858.896477:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896491:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971858.896501:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896507:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896514:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896518:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09800
00000400:00000200:3.0:1788971858.896525:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896531:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x10e5 [1] + 224
00000400:00000200:3.0:1788971858.896545:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971858.896550:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971858.896568:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353ce00 x1875872179066880/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971874 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971858.896623:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896628:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971858.896632:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896634:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896637:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896640:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09880
00000400:00000200:3.0:1788971858.896644:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971858.896649:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x10ed [1] + 224
00000400:00000200:3.0:1788971858.896659:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971858.896662:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971858.896669:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353e680 x1875872179067008/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971874 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971858.897574:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78.
00000400:00000200:1.0:1788971858.897582:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588088
00000400:00000010:1.0:1788971858.897585:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588088.
00000100:00000200:1.0:1788971858.897596:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353e680 x1875872179067008/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971874 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971858.897619:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353e680 x1875872179067008/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971874 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971858.897656:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971858.897659:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:1.0:1788971858.897661:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971858.897667:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353ce00 x1875872179066880/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971874 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971858.897675:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353ce00 x1875872179066880/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971874 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971858.897691:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2000.
02000000:00000010:1.0:1788971858.897700:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:1.0:1788971858.897717:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680.
02000000:00000010:1.0:1788971858.897723:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2e00.
02000000:00000010:1.0:1788971858.897725:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00.
00000100:00000010:1.0:1788971858.897728:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00.
00000020:00000010:1.0:1788971858.995712:0:110925:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971858.995830:0:110925:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:3.0:1788971859.591453:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9ff00.
00000100:00000010:2.0:1788971859.592344:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fc00.
00000020:00000010:2.0:1788971860.033015:0:110929:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166770.
00000020:00000010:2.0:1788971860.034804:0:110929:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166770.
00000020:00000010:2.0:1788971861.124295:0:110933:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166770.
00000020:00000010:2.0:1788971861.124426:0:110933:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166770.
00000020:00000010:0.0:1788971862.164810:0:110937:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971862.164909:0:110937:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000020:00000010:1.0:1788971863.255970:0:110941:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971863.256073:0:110941:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:2.0:1788971864.010361:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971864.010386:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ec00.
00000100:00000010:2.0:1788971864.010429:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410e00.
02000000:00000010:2.0:1788971864.010437:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e500.
02000000:00000010:3.0:1788971864.010559:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96200.
00000400:00000200:3.0:1788971864.010637:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971864.010643:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78b00.
00000400:00000010:3.0:1788971864.010650:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0.
00000100:00000200:3.0:1788971864.010669:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067136, portal 25
00000400:00000010:3.0:1788971864.010674:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480817f8.
00000100:00000200:3.0:1788971864.010677:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179067136, offset 0
00000400:00000200:3.0:1788971864.010686:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971864.010694:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.010701:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971864.010705:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971864.010709:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971864.010712:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971864.010714:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971864.010718:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [311:311:256:4294967295] 192.168.203.124@tcp LPNI seq info [311:311:8:4294967295]
00000400:00000200:3.0:1788971864.010722:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.010731:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971864.010737:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971864.010742:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00.
00000800:00000200:3.0:1788971864.010747:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971864.010754:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971864.010759:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971864.010777:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971864.010782:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a420.
00000100:00000010:3.0:1788971864.010793:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a8a0.
02000000:00000010:3.0:1788971864.010813:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97c00.
00000400:00000200:3.0:1788971864.010817:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971864.010819:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e788f0.
00000400:00000010:3.0:1788971864.010823:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0.
00000100:00000200:3.0:1788971864.010825:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067264, portal 10
00000400:00000010:3.0:1788971864.010830:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081c38.
00000100:00000200:3.0:1788971864.010832:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179067264, offset 0
00000400:00000200:3.0:1788971864.010836:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971864.010840:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.010843:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971864.010852:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971864.010855:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971864.010857:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971864.010860:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971864.010862:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [312:312:255:4294967295] 192.168.203.124@tcp LPNI seq info [312:312:7:4294967295]
00000400:00000200:3.0:1788971864.010866:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.010872:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971864.010876:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971864.010879:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500.
00000800:00000200:3.0:1788971864.010883:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971864.010886:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971864.010889:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971864.010895:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971864.010984:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971864.010989:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00.
00000400:00000200:2.0:1788971864.010995:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971864.011000:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971864.011004:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480817f8
00000400:00000010:2.0:1788971864.011007:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480817f8.
00000100:00000200:2.0:1788971864.011016:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179067136/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971864.011293:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971864.011296:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500.
00000400:00000200:2.0:1788971864.011321:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971864.011326:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971864.011329:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081c38
00000400:00000010:2.0:1788971864.011337:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081c38.
00000100:00000200:2.0:1788971864.011343:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410e00 x1875872179067264/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971880 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971864.012783:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.012790:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971864.012797:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.012801:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.012805:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.012809:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09900
00000400:00000200:3.0:1788971864.012814:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971864.012819:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x10f5 [1] + 224
00000400:00000200:3.0:1788971864.012833:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971864.012837:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971864.012848:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179067136/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971880 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971864.013080:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78b00.
00000400:00000200:2.0:1788971864.013086:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0
00000400:00000010:2.0:1788971864.013088:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0.
00000100:00000200:2.0:1788971864.013094:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179067136/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971880 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971864.013104:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179067136/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971880 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971864.013127:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96200.
02000000:00000010:2.0:1788971864.013133:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ec00.
00000100:00000010:2.0:1788971864.013138:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
00000400:00000200:2.0:1788971864.013941:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971864.013946:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971864.013952:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971864.013955:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971864.013959:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971864.013963:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09980
00000400:00000200:2.0:1788971864.013967:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971864.013972:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1105 [1] + 224
00000400:00000200:2.0:1788971864.013983:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971864.013988:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971864.013996:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410e00 x1875872179067264/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971880 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971864.014145:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e788f0.
00000400:00000200:3.0:1788971864.014150:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0
00000400:00000010:3.0:1788971864.014153:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0.
00000100:00000200:3.0:1788971864.014160:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410e00 x1875872179067264/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971880 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971864.014185:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410e00 x1875872179067264/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971880 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971864.014207:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97c00.
02000000:00000010:3.0:1788971864.014211:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e500.
00000100:00000010:3.0:1788971864.014215:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410e00.
00000020:00000010:0.0:1788971864.316542:0:110945:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971864.316643:0:110945:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000100:00000010:3.0:1788971864.711479:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a8a0.
00000100:00000010:2.0:1788971864.712762:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a420.
00000020:00000010:1.0:1788971865.360892:0:110949:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971865.361647:0:110949:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:0.0:1788971866.426726:0:110955:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971866.426804:0:110955:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000020:00000010:1.0:1788971867.476199:0:110959:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971867.476328:0:110959:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971868.520126:0:110964:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971868.520208:0:110964:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:0.0:1788971869.130937:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380.
02000000:00000010:0.0:1788971869.130958:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500.
02000000:00000010:1.0:1788971869.131107:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3000.
00000400:00000200:1.0:1788971869.131117:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971869.131122:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971869.131129:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971869.131135:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067392, portal 25
00000400:00000010:1.0:1788971869.131140:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588088.
00000100:00000200:1.0:1788971869.131144:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179067392, offset 0
00000400:00000200:1.0:1788971869.131152:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971869.131160:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971869.131168:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971869.131173:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971869.131178:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971869.131182:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971869.131185:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000100:00000010:0.0:1788971869.131185:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800.
00000400:00000200:1.0:1788971869.131192:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [313:313:256:4294967295] 192.168.203.124@tcp LPNI seq info [313:313:8:4294967295]
02000000:00000010:0.0:1788971869.131194:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55f00.
00000400:00000200:1.0:1788971869.131199:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971869.131213:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971869.131220:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
02000000:00000010:0.0:1788971869.131226:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016c00.
00000800:00000010:1.0:1788971869.131227:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637100.
00000400:00000200:0.0:1788971869.131235:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:1.0:1788971869.131238:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971869.131246:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971869.131251:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971869.131270:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971869.131275:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f180.
00000100:00000010:1.0:1788971869.131285:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9ff00.
00000800:00000200:3.0:1788971869.131530:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971869.131544:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637100.
00000400:00000200:3.0:1788971869.131550:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971869.131556:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971869.131560:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588088
00000400:00000010:3.0:1788971869.131563:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588088.
00000100:00000200:3.0:1788971869.131571:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c380 x1875872179067392/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971869.131632:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163ec8.
00000400:00000010:0.0:1788971869.131637:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198.
00000100:00000200:0.0:1788971869.131641:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067520, portal 10
00000400:00000010:0.0:1788971869.131646:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990.
00000100:00000200:0.0:1788971869.131648:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179067520, offset 0
00000400:00000200:0.0:1788971869.131654:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971869.131659:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971869.131664:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971869.131668:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971869.131670:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971869.131672:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971869.131683:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971869.131686:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [314:314:256:4294967295] 192.168.203.124@tcp LPNI seq info [314:314:8:4294967295]
00000400:00000200:0.0:1788971869.131690:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971869.131697:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971869.131701:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971869.131705:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800.
00000800:00000200:0.0:1788971869.131709:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971869.131713:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971869.131716:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971869.131735:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971869.131823:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971869.131827:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800.
00000400:00000200:3.0:1788971869.131830:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971869.131834:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971869.131837:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990
00000400:00000010:3.0:1788971869.131838:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990.
00000100:00000200:3.0:1788971869.131844:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f800 x1875872179067520/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971885 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971869.134245:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134254:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971869.134262:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134266:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134271:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134275:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09a00
00000400:00000200:3.0:1788971869.134334:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134351:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1115 [1] + 224
00000400:00000200:3.0:1788971869.134363:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971869.134368:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971869.134380:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c380 x1875872179067392/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971885 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971869.134418:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134421:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971869.134425:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134428:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134431:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134433:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09a80
00000400:00000200:3.0:1788971869.134437:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971869.134441:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x111d [1] + 224
00000400:00000200:3.0:1788971869.134448:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971869.134451:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971869.134465:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f800 x1875872179067520/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971885 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971869.134546:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163ec8.
00000400:00000200:0.0:1788971869.134552:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198
00000400:00000010:0.0:1788971869.134555:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198.
00000100:00000200:0.0:1788971869.134562:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f800 x1875872179067520/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971885 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971869.134572:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f800 x1875872179067520/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971885 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971869.134602:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016c00.
02000000:00000010:0.0:1788971869.134607:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55f00.
00000100:00000010:0.0:1788971869.134612:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800.
00000400:00000010:1.0:1788971869.142501:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971869.142512:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:1.0:1788971869.142515:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971869.142760:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c380 x1875872179067392/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971885 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971869.142772:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c380 x1875872179067392/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971885 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971869.142803:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3000.
02000000:00000010:1.0:1788971869.142811:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500.
00000100:00000010:1.0:1788971869.142816:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380.
00000020:00000010:1.0:1788971869.560414:0:110969:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971869.560500:0:110969:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:3.0:1788971869.831379:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9ff00.
00000100:00000010:2.0:1788971869.831451:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f180.
00000020:00000010:1.0:1788971870.589460:0:110973:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971870.607132:0:110973:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:3.0:1788971871.634943:0:110977:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971871.635032:0:110977:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:2.0:1788971872.673419:0:110981:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971872.673529:0:110981:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000020:00000010:1.0:1788971873.714780:0:110988:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971873.714868:0:110988:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:2.0:1788971874.249115:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971874.249150:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb496ee000.
00000100:00000010:2.0:1788971874.249187:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412680.
02000000:00000010:2.0:1788971874.249195:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381d800.
02000000:00000010:3.0:1788971874.249367:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97a00.
00000400:00000200:3.0:1788971874.249378:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971874.249382:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e788f0.
00000400:00000010:3.0:1788971874.249389:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0.
00000100:00000200:3.0:1788971874.249395:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067648, portal 25
00000400:00000010:3.0:1788971874.249400:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110.
00000100:00000200:3.0:1788971874.249403:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179067648, offset 0
00000400:00000200:3.0:1788971874.249411:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971874.249419:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.249426:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971874.249436:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971874.249441:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971874.249445:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971874.249447:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971874.249451:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [315:315:256:4294967295] 192.168.203.124@tcp LPNI seq info [315:315:8:4294967295]
00000400:00000200:3.0:1788971874.249455:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.249465:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971874.249470:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971874.249475:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900.
00000800:00000200:3.0:1788971874.249481:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971874.249488:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971874.249493:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971874.249524:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971874.249529:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347aae0.
00000100:00000010:3.0:1788971874.249539:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347a1e0.
02000000:00000010:3.0:1788971874.249558:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97200.
00000400:00000200:3.0:1788971874.249563:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971874.249565:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78318.
00000400:00000010:3.0:1788971874.249569:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081330.
00000100:00000200:3.0:1788971874.249571:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067776, portal 10
00000400:00000010:3.0:1788971874.249576:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480813b8.
00000100:00000200:3.0:1788971874.249578:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179067776, offset 0
00000400:00000200:3.0:1788971874.249582:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971874.249586:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.249589:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971874.249591:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971874.249594:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971874.249596:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971874.249598:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971874.249601:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [316:316:255:4294967295] 192.168.203.124@tcp LPNI seq info [316:316:7:4294967295]
00000400:00000200:3.0:1788971874.249605:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.249617:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971874.249622:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971874.249625:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500.
00000800:00000200:3.0:1788971874.249629:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971874.249633:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971874.249636:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971874.249642:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971874.250110:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971874.250115:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900.
00000400:00000200:2.0:1788971874.250121:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971874.250126:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971874.250131:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110
00000400:00000010:2.0:1788971874.250134:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110.
00000100:00000200:2.0:1788971874.250144:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179067648/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971874.250219:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971874.250222:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500.
00000400:00000200:2.0:1788971874.250225:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971874.250237:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971874.250241:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480813b8
00000400:00000010:2.0:1788971874.250242:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480813b8.
00000100:00000200:2.0:1788971874.250248:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412680 x1875872179067776/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971890 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971874.252224:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252233:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971874.252240:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252244:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252249:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252253:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09b00
00000400:00000200:3.0:1788971874.252257:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252263:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1125 [1] + 224
00000400:00000200:3.0:1788971874.252272:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971874.252276:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971874.252286:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179067648/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971890 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971874.252376:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252380:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971874.252384:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252387:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252389:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252392:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09b80
00000400:00000200:3.0:1788971874.252395:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971874.252399:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1135 [1] + 224
00000400:00000200:3.0:1788971874.252406:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971874.252410:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971874.252429:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412680 x1875872179067776/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971890 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971874.252637:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78318.
00000400:00000200:2.0:1788971874.252643:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081330
00000400:00000010:2.0:1788971874.252645:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081330.
00000100:00000200:2.0:1788971874.252653:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412680 x1875872179067776/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971890 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971874.252664:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412680 x1875872179067776/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971890 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971874.252692:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e788f0.
00000400:00000200:2.0:1788971874.252694:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0
00000400:00000010:2.0:1788971874.252696:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0.
00000100:00000200:2.0:1788971874.252700:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179067648/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971890 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971874.252720:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179067648/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971890 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971874.252736:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97200.
02000000:00000010:2.0:1788971874.252743:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381d800.
00000100:00000010:2.0:1788971874.252748:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412680.
02000000:00000010:2.0:1788971874.252751:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97a00.
02000000:00000010:2.0:1788971874.252754:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb496ee000.
00000100:00000010:2.0:1788971874.252758:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
00000020:00000010:2.0:1788971874.765253:0:110992:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971874.765828:0:110992:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000100:00000010:3.0:1788971874.951432:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347a1e0.
00000100:00000010:2.0:1788971874.952479:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347aae0.
00000020:00000010:1.0:1788971875.802779:0:110996:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971875.802868:0:110996:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:3.0:1788971876.840154:0:111000:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971876.840239:0:111000:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971877.870114:0:111004:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971877.870219:0:111004:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971878.919250:0:111008:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971878.919618:0:111008:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000100:00000010:2.0:1788971879.367481:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971879.367505:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381d800.
00000100:00000010:2.0:1788971879.367553:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412680.
02000000:00000010:2.0:1788971879.367558:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381dd00.
02000000:00000010:2.0:1788971879.367594:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d68c00.
00000400:00000200:2.0:1788971879.367603:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971879.367607:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3b58.
00000400:00000010:2.0:1788971879.367614:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0770.
00000100:00000200:2.0:1788971879.367621:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068032, portal 10
00000400:00000010:2.0:1788971879.367628:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971879.367631:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179068032, offset 0
00000400:00000200:2.0:1788971879.367640:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971879.367649:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971879.367657:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971879.367661:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971879.367665:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971879.367669:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971879.367672:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971879.367676:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [317:317:256:4294967295] 192.168.203.124@tcp LPNI seq info [317:317:8:4294967295]
00000400:00000200:2.0:1788971879.367697:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971879.367707:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971879.367713:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971879.367718:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d200.
00000800:00000200:2.0:1788971879.367725:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971879.367734:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971879.367739:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971879.367756:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971879.367761:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7420.
00000100:00000010:2.0:1788971879.367770:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7d20.
02000000:00000010:2.0:1788971879.367795:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d68000.
00000400:00000200:2.0:1788971879.367799:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971879.367801:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3580.
00000400:00000010:2.0:1788971879.367805:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38.
00000100:00000200:2.0:1788971879.367808:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179067904, portal 25
00000400:00000010:2.0:1788971879.367812:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0aa0.
00000100:00000200:2.0:1788971879.367814:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179067904, offset 0
00000400:00000200:2.0:1788971879.367818:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971879.367821:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971879.367836:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971879.367845:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971879.367848:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971879.367851:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971879.367853:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971879.367855:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [318:318:255:4294967295] 192.168.203.124@tcp LPNI seq info [318:318:7:4294967295]
00000400:00000200:2.0:1788971879.367859:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971879.367864:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971879.367868:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971879.367873:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb496ee000.
00000800:00000200:2.0:1788971879.367877:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971879.367888:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971879.367891:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb496ee000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971879.367896:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971879.367943:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971879.367947:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d200.
00000400:00000200:2.0:1788971879.367954:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971879.367959:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971879.367964:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330
00000400:00000010:2.0:1788971879.367966:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971879.367975:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412680 x1875872179068032/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971879.368072:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971879.368075:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb496ee000.
00000400:00000200:2.0:1788971879.368077:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971879.368081:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971879.368084:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0aa0
00000400:00000010:2.0:1788971879.368086:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0aa0.
00000100:00000200:2.0:1788971879.368090:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179067904/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971895 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971879.376988:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.376997:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971879.377005:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377010:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377622:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377628:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09c00
00000400:00000200:3.0:1788971879.377633:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377639:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1155 [1] + 224
00000400:00000200:3.0:1788971879.377656:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971879.377661:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971879.377676:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179067904/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971895 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971879.377716:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377720:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971879.377724:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377727:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377729:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377732:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09c80
00000400:00000200:3.0:1788971879.377735:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971879.377739:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1145 [1] + 224
00000400:00000200:3.0:1788971879.377747:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971879.377765:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971879.377772:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412680 x1875872179068032/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971895 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971879.378510:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3580.
00000400:00000200:2.0:1788971879.378518:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38
00000400:00000010:2.0:1788971879.378520:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38.
00000100:00000200:2.0:1788971879.378528:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179067904/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971895 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971879.378537:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179067904/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971895 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971879.378561:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3b58.
00000400:00000200:2.0:1788971879.378564:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0770
00000400:00000010:2.0:1788971879.378566:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0770.
00000100:00000200:2.0:1788971879.378570:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412680 x1875872179068032/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971895 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971879.378593:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412680 x1875872179068032/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971895 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971879.378616:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d68000.
02000000:00000010:2.0:1788971879.378622:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381d800.
00000100:00000010:2.0:1788971879.378627:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971879.378631:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d68c00.
02000000:00000010:2.0:1788971879.378633:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381dd00.
00000100:00000010:2.0:1788971879.378635:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412680.
00000020:00000010:2.0:1788971879.968291:0:111013:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971879.968418:0:111013:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000100:00000010:3.0:1788971880.071617:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7d20.
00000100:00000010:2.0:1788971880.074891:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7420.
00000020:00000010:3.0:1788971881.017364:0:111017:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971881.018570:0:111017:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:0.0:1788971882.093465:0:111021:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971882.093551:0:111021:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971883.136070:0:111025:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971883.136157:0:111025:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971884.161059:0:111029:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94e80.
00000020:00000010:0.0:1788971884.161140:0:111029:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94e80.
00000100:00000010:0.0:1788971884.489346:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800.
02000000:00000010:0.0:1788971884.489378:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00.
00000100:00000010:0.0:1788971884.489405:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971884.489410:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
02000000:00000010:0.0:1788971884.489472:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016a00.
00000400:00000200:0.0:1788971884.489483:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971884.489487:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163ec8.
00000400:00000010:0.0:1788971884.489493:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198.
00000100:00000200:0.0:1788971884.489500:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068288, portal 10
00000400:00000010:0.0:1788971884.489507:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084ee0.
00000100:00000200:0.0:1788971884.489511:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179068288, offset 0
00000400:00000200:0.0:1788971884.489519:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971884.489528:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971884.489545:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971884.489551:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971884.489555:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971884.489559:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971884.489562:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971884.489566:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [319:319:256:4294967295] 192.168.203.124@tcp LPNI seq info [319:319:8:4294967295]
00000400:00000200:0.0:1788971884.489570:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971884.489580:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971884.489585:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971884.489590:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55400.
00000800:00000200:0.0:1788971884.489597:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971884.489605:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971884.489609:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971884.489631:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971884.489636:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969840.
00000100:00000010:0.0:1788971884.489648:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969000.
02000000:00000010:0.0:1788971884.489674:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017200.
00000400:00000200:0.0:1788971884.489678:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971884.489680:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163210.
00000400:00000010:0.0:1788971884.489684:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084d48.
00000100:00000200:0.0:1788971884.489687:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068160, portal 25
00000400:00000010:0.0:1788971884.489690:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480842a8.
00000100:00000200:0.0:1788971884.489698:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179068160, offset 0
00000400:00000200:0.0:1788971884.489702:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971884.489711:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971884.489716:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971884.489718:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971884.489721:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971884.489723:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971884.489725:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971884.489728:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [320:320:255:4294967295] 192.168.203.124@tcp LPNI seq info [320:320:7:4294967295]
00000400:00000200:0.0:1788971884.489731:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971884.489737:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971884.489741:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971884.489745:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00.
00000800:00000200:0.0:1788971884.489749:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971884.489754:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971884.489756:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971884.489760:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971884.490009:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971884.490013:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55400.
00000400:00000200:2.0:1788971884.490029:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971884.490034:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971884.490040:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084ee0
00000400:00000010:2.0:1788971884.490042:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084ee0.
00000100:00000200:2.0:1788971884.490050:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179068288/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971884.490123:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971884.490126:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00.
00000400:00000200:2.0:1788971884.490129:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971884.490132:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971884.490135:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480842a8
00000400:00000010:2.0:1788971884.490136:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480842a8.
00000100:00000200:2.0:1788971884.490140:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f800 x1875872179068160/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971900 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971884.492226:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971884.492239:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971884.492250:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971884.492257:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971884.492263:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971884.492269:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09d00
00000400:00000200:3.0:1788971884.492275:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971884.492283:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x116d [1] + 224
00000400:00000200:3.0:1788971884.492345:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971884.492351:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971884.492369:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f800 x1875872179068160/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971900 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971884.492660:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163210.
00000400:00000200:0.0:1788971884.492672:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084d48
00000400:00000010:0.0:1788971884.492675:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084d48.
00000100:00000200:0.0:1788971884.492688:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f800 x1875872179068160/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971900 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971884.492700:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f800 x1875872179068160/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971900 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971884.492731:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017200.
02000000:00000010:0.0:1788971884.492738:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00.
00000100:00000010:0.0:1788971884.492743:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800.
00000400:00000200:2.0:1788971884.493373:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971884.493379:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971884.493389:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971884.493405:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971884.493411:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971884.493415:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09d80
00000400:00000200:2.0:1788971884.493420:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971884.493432:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1165 [1] + 224
00000400:00000200:2.0:1788971884.493449:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971884.493453:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971884.493463:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179068288/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971900 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971884.493674:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163ec8.
00000400:00000200:0.0:1788971884.493678:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198
00000400:00000010:0.0:1788971884.493680:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198.
00000100:00000200:0.0:1788971884.493686:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179068288/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971900 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971884.493701:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179068288/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971900 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971884.493724:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016a00.
02000000:00000010:0.0:1788971884.493728:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971884.493731:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000020:00000010:1.0:1788971885.190226:0:111033:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971885.190791:0:111033:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:2.0:1788971885.191938:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969840.
00000100:00000010:3.0:1788971885.192359:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969000.
00000020:00000010:3.0:1788971886.231903:0:111038:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971886.232000:0:111038:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:1.0:1788971887.266492:0:111042:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971887.266621:0:111042:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971888.309997:0:111046:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971888.310145:0:111046:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:0.0:1788971889.351535:0:111050:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d947e0.
00000020:00000010:0.0:1788971889.351646:0:111050:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d947e0.
00000100:00000010:2.0:1788971889.608583:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412680.
02000000:00000010:2.0:1788971889.608614:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb496ee000.
00000100:00000010:2.0:1788971889.608662:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971889.608671:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381dd00.
02000000:00000010:2.0:1788971889.608718:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d68e00.
00000400:00000200:2.0:1788971889.608745:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971889.608750:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3b58.
00000400:00000010:2.0:1788971889.608759:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0770.
00000100:00000200:2.0:1788971889.608768:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068544, portal 10
00000400:00000010:2.0:1788971889.608776:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38.
00000100:00000200:2.0:1788971889.608780:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179068544, offset 0
00000400:00000200:2.0:1788971889.608791:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971889.608801:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.608811:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971889.608818:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971889.608823:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971889.608827:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971889.608830:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971889.608835:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [321:321:256:4294967295] 192.168.203.124@tcp LPNI seq info [321:321:8:4294967295]
00000400:00000200:2.0:1788971889.608840:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.608862:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971889.608887:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971889.608892:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d800.
00000800:00000200:2.0:1788971889.608901:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971889.608909:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971889.608915:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971889.608948:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971889.608955:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0.
00000100:00000010:2.0:1788971889.608968:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7900.
02000000:00000010:2.0:1788971889.608994:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69000.
00000400:00000200:2.0:1788971889.608999:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971889.609001:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3580.
00000400:00000010:2.0:1788971889.609005:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0aa0.
00000100:00000200:2.0:1788971889.609008:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068416, portal 25
00000400:00000010:2.0:1788971889.609012:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971889.609014:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179068416, offset 0
00000400:00000200:2.0:1788971889.609019:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971889.609023:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.609026:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971889.609029:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971889.609031:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971889.609034:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971889.609036:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971889.609039:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [322:322:255:4294967295] 192.168.203.124@tcp LPNI seq info [322:322:7:4294967295]
00000400:00000200:2.0:1788971889.609043:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.609048:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971889.609053:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971889.609056:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d200.
00000800:00000200:2.0:1788971889.609060:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971889.609064:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971889.609067:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971889.609085:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971889.609151:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971889.609156:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d800.
00000400:00000200:2.0:1788971889.609161:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971889.609167:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971889.609174:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38
00000400:00000010:2.0:1788971889.609176:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38.
00000100:00000200:2.0:1788971889.609186:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179068544/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971889.609271:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971889.609274:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d200.
00000400:00000200:2.0:1788971889.609276:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971889.609280:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971889.609283:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330
00000400:00000010:2.0:1788971889.609285:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971889.609290:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412680 x1875872179068416/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971905 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971889.611254:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.611265:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971889.611275:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.611280:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.611286:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.611292:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09e00
00000400:00000200:2.0:1788971889.611297:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.611338:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1185 [1] + 224
00000400:00000200:2.0:1788971889.611355:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971889.611360:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971889.611378:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412680 x1875872179068416/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971905 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971889.611764:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3580.
00000400:00000200:3.0:1788971889.611773:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0aa0
00000400:00000010:3.0:1788971889.611775:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0aa0.
00000100:00000200:3.0:1788971889.611783:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412680 x1875872179068416/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971905 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971889.611792:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412680 x1875872179068416/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971905 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971889.611826:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69000.
02000000:00000010:3.0:1788971889.612167:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb496ee000.
00000100:00000010:3.0:1788971889.612177:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412680.
00000400:00000200:2.0:1788971889.612717:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.612724:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971889.612763:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.612767:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.612772:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.612776:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09e80
00000400:00000200:2.0:1788971889.612782:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971889.612788:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1175 [1] + 224
00000400:00000200:2.0:1788971889.612802:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971889.612807:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971889.612818:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179068544/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971905 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971889.613151:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3b58.
00000400:00000200:3.0:1788971889.613157:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0770
00000400:00000010:3.0:1788971889.613160:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0770.
00000100:00000200:3.0:1788971889.613167:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179068544/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971905 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971889.613176:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179068544/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971905 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971889.613206:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d68e00.
02000000:00000010:3.0:1788971889.613212:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381dd00.
00000100:00000010:3.0:1788971889.613216:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
00000100:00000010:2.0:1788971890.311463:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0.
00000100:00000010:3.0:1788971890.311847:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7900.
00000020:00000010:2.0:1788971890.407522:0:111054:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971890.407628:0:111054:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000020:00000010:3.0:1788971891.477119:0:111058:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971891.477223:0:111058:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:0.0:1788971892.527847:0:111062:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d947e0.
00000020:00000010:0.0:1788971892.527941:0:111062:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d947e0.
00000020:00000010:3.0:1788971893.575856:0:111067:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971893.576033:0:111067:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:1.0:1788971894.640922:0:111071:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971894.641011:0:111071:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:2.0:1788971894.727672:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411180.
02000000:00000010:2.0:1788971894.727694:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381d200.
00000100:00000010:2.0:1788971894.727732:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412300.
02000000:00000010:2.0:1788971894.727737:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381d800.
02000000:00000010:3.0:1788971894.727951:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97000.
00000400:00000200:3.0:1788971894.727961:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971894.727965:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78688.
00000400:00000010:3.0:1788971894.727972:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081000.
00000100:00000200:3.0:1788971894.727980:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068672, portal 25
00000400:00000010:3.0:1788971894.727984:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440.
00000100:00000200:3.0:1788971894.727987:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179068672, offset 0
00000400:00000200:3.0:1788971894.727996:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971894.728004:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971894.728012:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971894.728017:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971894.728020:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971894.728024:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971894.728027:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971894.728045:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [323:323:256:4294967295] 192.168.203.124@tcp LPNI seq info [323:323:8:4294967295]
00000400:00000200:3.0:1788971894.728050:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971894.728059:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971894.728066:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971894.728071:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00.
00000800:00000200:3.0:1788971894.728078:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971894.728085:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971894.728088:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971894.728158:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971894.728164:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347ad20.
00000100:00000010:3.0:1788971894.728173:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347a900.
02000000:00000010:3.0:1788971894.728192:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96e00.
00000400:00000200:3.0:1788971894.728197:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971894.728200:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78bb0.
00000400:00000010:3.0:1788971894.728203:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788971894.728206:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068800, portal 10
00000400:00000010:3.0:1788971894.728211:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081cc0.
00000100:00000200:3.0:1788971894.728213:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179068800, offset 0
00000400:00000200:3.0:1788971894.728218:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971894.728221:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971894.728225:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971894.728228:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971894.728230:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971894.728232:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971894.728234:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971894.728237:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [324:324:255:4294967295] 192.168.203.124@tcp LPNI seq info [324:324:7:4294967295]
00000400:00000200:3.0:1788971894.728241:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971894.728247:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971894.728261:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971894.728265:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00.
00000800:00000200:3.0:1788971894.728276:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971894.728281:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971894.728283:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971894.728290:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971894.729365:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971894.729373:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00.
00000400:00000200:2.0:1788971894.729380:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971894.729387:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971894.729671:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440
00000400:00000010:2.0:1788971894.729674:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440.
00000100:00000200:2.0:1788971894.729685:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43411180 x1875872179068672/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971894.729759:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971894.729762:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00.
00000400:00000200:2.0:1788971894.729765:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971894.729769:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971894.729771:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081cc0
00000400:00000010:2.0:1788971894.729773:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081cc0.
00000100:00000200:2.0:1788971894.729788:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412300 x1875872179068800/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971910 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971894.732514:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732522:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971894.732530:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732534:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732539:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732543:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09f00
00000400:00000200:2.0:1788971894.732549:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732555:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1195 [1] + 224
00000400:00000200:2.0:1788971894.732566:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971894.732570:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971894.732584:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43411180 x1875872179068672/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971910 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971894.732614:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732618:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971894.732622:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732624:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732627:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732629:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09f80
00000400:00000200:2.0:1788971894.732633:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971894.732636:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x11a5 [1] + 224
00000400:00000200:2.0:1788971894.732658:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971894.732662:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971894.732669:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412300 x1875872179068800/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971910 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971894.732904:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78bb0.
00000400:00000200:3.0:1788971894.732910:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081088
00000400:00000010:3.0:1788971894.732912:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788971894.732919:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412300 x1875872179068800/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971910 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971894.732927:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412300 x1875872179068800/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971910 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971894.732955:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78688.
00000400:00000200:3.0:1788971894.732958:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081000
00000400:00000010:3.0:1788971894.732959:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081000.
00000100:00000200:3.0:1788971894.732963:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43411180 x1875872179068672/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971910 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971894.732970:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43411180 x1875872179068672/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971910 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971894.732983:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96e00.
02000000:00000010:3.0:1788971894.732987:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381d800.
00000100:00000010:3.0:1788971894.732992:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412300.
02000000:00000010:3.0:1788971894.732995:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97000.
02000000:00000010:3.0:1788971894.732997:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381d200.
00000100:00000010:3.0:1788971894.732999:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411180.
00000100:00000010:2.0:1788971895.431457:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347ad20.
00000100:00000010:3.0:1788971895.431901:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347a900.
00000020:00000010:0.0:1788971895.661586:0:111075:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d947e0.
00000020:00000010:0.0:1788971895.661674:0:111075:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d947e0.
00000020:00000010:1.0:1788971896.712805:0:111079:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971896.717144:0:111079:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:2.0:1788971897.751032:0:111083:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971897.751124:0:111083:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000020:00000010:3.0:1788971898.783337:0:111087:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971898.783532:0:111087:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000020:00000010:2.0:1788971899.819072:0:111091:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971899.819155:0:111091:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000100:00000010:0.0:1788971899.847570:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971899.847592:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971899.847633:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800.
02000000:00000010:0.0:1788971899.847638:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00.
02000000:00000010:0.0:1788971899.847669:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017200.
00000400:00000200:0.0:1788971899.847678:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971899.847682:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163ec8.
00000400:00000010:0.0:1788971899.847688:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198.
00000100:00000200:0.0:1788971899.847695:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069056, portal 10
00000400:00000010:0.0:1788971899.847702:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084d48.
00000100:00000200:0.0:1788971899.847705:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179069056, offset 0
00000400:00000200:0.0:1788971899.847713:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971899.847721:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971899.847729:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971899.847734:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971899.847738:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971899.847742:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971899.847744:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971899.847748:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [325:325:256:4294967295] 192.168.203.124@tcp LPNI seq info [325:325:8:4294967295]
00000400:00000200:0.0:1788971899.847764:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971899.847775:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971899.847782:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971899.847787:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000.
00000800:00000200:0.0:1788971899.847793:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971899.847801:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971899.847805:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971899.847825:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971899.847831:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969a20.
00000100:00000010:0.0:1788971899.847844:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449695a0.
02000000:00000010:0.0:1788971899.847873:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017a00.
00000400:00000200:0.0:1788971899.847878:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971899.847880:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163210.
00000400:00000010:0.0:1788971899.847884:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084a18.
00000100:00000200:0.0:1788971899.847887:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179068928, portal 25
00000400:00000010:0.0:1788971899.847891:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480847f8.
00000100:00000200:0.0:1788971899.847893:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179068928, offset 0
00000400:00000200:0.0:1788971899.847898:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971899.847901:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971899.847904:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971899.847907:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971899.847909:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971899.847912:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971899.847914:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971899.847917:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [326:326:255:4294967295] 192.168.203.124@tcp LPNI seq info [326:326:7:4294967295]
00000400:00000200:0.0:1788971899.847920:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971899.847926:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971899.847931:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971899.847935:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00.
00000800:00000200:0.0:1788971899.847939:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971899.847949:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971899.847951:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971899.847955:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971899.848134:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971899.848139:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000.
00000400:00000200:2.0:1788971899.848145:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971899.848150:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971899.848155:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084d48
00000400:00000010:2.0:1788971899.848158:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084d48.
00000100:00000200:2.0:1788971899.848166:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f800 x1875872179069056/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971899.848236:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971899.848239:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00.
00000400:00000200:2.0:1788971899.848242:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971899.848246:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971899.848249:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480847f8
00000400:00000010:2.0:1788971899.848250:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480847f8.
00000100:00000200:2.0:1788971899.848255:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179068928/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971915 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971899.851896:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971899.851905:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971899.851912:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971899.851916:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971899.851921:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971899.851924:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a080
00000400:00000200:2.0:1788971899.851929:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971899.851934:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x11b5 [1] + 224
00000400:00000200:2.0:1788971899.851948:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971899.851952:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971899.851965:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f800 x1875872179069056/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971915 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971899.853732:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971899.853739:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971899.853746:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971899.853763:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971899.853767:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971899.853771:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a000
00000400:00000200:3.0:1788971899.853776:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971899.853782:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x11bd [1] + 224
00000400:00000200:3.0:1788971899.853814:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971899.853819:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971899.853831:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179068928/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971915 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971899.860764:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163ec8.
00000400:00000200:1.0:1788971899.860776:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198
00000400:00000010:1.0:1788971899.860779:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198.
00000100:00000200:1.0:1788971899.860965:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f800 x1875872179069056/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971915 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971899.860978:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f800 x1875872179069056/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971915 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971899.861027:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017200.
02000000:00000010:1.0:1788971899.861035:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00.
00000100:00000010:1.0:1788971899.861041:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800.
00000400:00000010:0.0:1788971899.861171:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163210.
00000400:00000200:0.0:1788971899.861176:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084a18
00000400:00000010:0.0:1788971899.861177:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084a18.
00000100:00000200:0.0:1788971899.861184:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179068928/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971915 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971899.861193:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179068928/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971915 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971899.861205:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017a00.
02000000:00000010:0.0:1788971899.861208:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971899.861210:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000100:00000010:3.0:1788971900.551430:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449695a0.
00000100:00000010:2.0:1788971900.552214:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969a20.
00000020:00000010:1.0:1788971900.844716:0:111096:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971900.844802:0:111096:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:0.0:1788971901.873034:0:111100:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d947e0.
00000020:00000010:0.0:1788971901.873126:0:111100:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d947e0.
00000020:00000010:2.0:1788971902.893747:0:111104:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971902.893828:0:111104:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000020:00000010:1.0:1788971903.931041:0:111108:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971903.931129:0:111108:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971904.965170:0:111112:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971904.965276:0:111112:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:0.0:1788971904.968555:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971904.968578:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971904.968606:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700.
02000000:00000010:0.0:1788971904.968612:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400.
02000000:00000010:0.0:1788971904.968651:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017e00.
00000400:00000200:0.0:1788971904.968668:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971904.968673:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163210.
00000400:00000010:0.0:1788971904.968680:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084a18.
00000100:00000200:0.0:1788971904.968686:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069184, portal 25
00000400:00000010:0.0:1788971904.968691:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084f68.
00000100:00000200:0.0:1788971904.968695:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179069184, offset 0
00000400:00000200:0.0:1788971904.968704:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971904.968713:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971904.968720:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971904.968725:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971904.968729:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971904.968741:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971904.968744:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971904.968748:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [327:327:256:4294967295] 192.168.203.124@tcp LPNI seq info [327:327:8:4294967295]
00000400:00000200:0.0:1788971904.968753:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971904.968763:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971904.968770:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971904.968774:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800.
00000800:00000200:0.0:1788971904.968781:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971904.968788:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971904.968793:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971904.968856:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971904.968862:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969180.
00000100:00000010:0.0:1788971904.968872:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969480.
02000000:00000010:0.0:1788971904.968891:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016600.
00000400:00000200:0.0:1788971904.968896:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971904.968899:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163630.
00000400:00000010:0.0:1788971904.968903:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480844c8.
00000100:00000200:0.0:1788971904.968906:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069312, portal 10
00000400:00000010:0.0:1788971904.968911:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084e58.
00000100:00000200:0.0:1788971904.968913:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179069312, offset 0
00000400:00000200:0.0:1788971904.968918:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971904.968921:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971904.968924:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971904.968937:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971904.968940:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971904.968943:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971904.968945:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971904.968948:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [328:328:255:4294967295] 192.168.203.124@tcp LPNI seq info [328:328:7:4294967295]
00000400:00000200:0.0:1788971904.968951:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971904.968958:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971904.968975:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971904.968979:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500.
00000800:00000200:0.0:1788971904.968983:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971904.968988:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971904.968991:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971904.968997:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971904.972934:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971904.972943:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800.
00000400:00000200:3.0:1788971904.972950:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971904.972959:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971904.972977:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084f68
00000400:00000010:3.0:1788971904.972979:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084f68.
00000100:00000200:3.0:1788971904.972991:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179069184/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971904.973076:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971904.973080:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500.
00000400:00000200:3.0:1788971904.973085:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971904.973089:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971904.973091:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084e58
00000400:00000010:3.0:1788971904.973094:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084e58.
00000100:00000200:3.0:1788971904.973523:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c700 x1875872179069312/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971920 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971904.986029:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.986040:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971904.986049:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.986053:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.986060:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.986064:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a100
00000400:00000200:2.0:1788971904.986079:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.986086:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x11c5 [1] + 224
00000400:00000200:2.0:1788971904.986104:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971904.986109:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971904.986123:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179069184/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971920 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971904.986210:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163210.
00000400:00000200:0.0:1788971904.986215:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084a18
00000400:00000010:0.0:1788971904.986217:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084a18.
00000100:00000200:0.0:1788971904.986223:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179069184/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971920 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971904.986232:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179069184/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971920 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971904.986261:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017e00.
02000000:00000010:0.0:1788971904.986267:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971904.986272:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000400:00000200:2.0:1788971904.987838:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.987846:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971904.987853:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.987858:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.987862:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.987867:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a180
00000400:00000200:2.0:1788971904.987871:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971904.987877:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x11cd [1] + 224
00000400:00000200:2.0:1788971904.987892:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971904.987896:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971904.987907:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c700 x1875872179069312/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971920 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971904.988040:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163630.
00000400:00000200:0.0:1788971904.988047:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480844c8
00000400:00000010:0.0:1788971904.988049:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480844c8.
00000100:00000200:0.0:1788971904.988056:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c700 x1875872179069312/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971920 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971904.988065:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c700 x1875872179069312/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971920 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971904.988092:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016600.
02000000:00000010:0.0:1788971904.988097:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400.
00000100:00000010:0.0:1788971904.988102:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700.
00000100:00000010:3.0:1788971905.671491:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969480.
00000100:00000010:2.0:1788971905.671948:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969180.
00000020:00000010:0.0:1788971906.014934:0:111116:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d947e0.
00000020:00000010:0.0:1788971906.015023:0:111116:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d947e0.
00000020:00000010:2.0:1788971907.089169:0:111120:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971907.089256:0:111120:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971908.119794:0:111125:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166fa0.
00000020:00000010:2.0:1788971908.120086:0:111125:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166fa0.
00000020:00000010:1.0:1788971909.168804:0:111129:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ba0.
00000020:00000010:1.0:1788971909.168911:0:111129:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ba0.
00000100:00000010:3.0:1788971910.087499:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00.
02000000:00000010:3.0:1788971910.087522:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00.
00000100:00000010:3.0:1788971910.087557:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480.
02000000:00000010:3.0:1788971910.087561:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400.
02000000:00000010:2.0:1788971910.087754:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69200.
00000400:00000200:2.0:1788971910.087764:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971910.087769:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b39f8.
00000400:00000010:2.0:1788971910.087775:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330.
00000100:00000200:2.0:1788971910.087782:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069440, portal 25
00000400:00000010:2.0:1788971910.087787:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38.
00000100:00000200:2.0:1788971910.087790:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179069440, offset 0
00000400:00000200:2.0:1788971910.087798:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971910.087806:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.087813:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971910.087818:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971910.087822:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971910.087826:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971910.087829:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971910.087832:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [329:329:256:4294967295] 192.168.203.124@tcp LPNI seq info [329:329:8:4294967295]
00000400:00000200:2.0:1788971910.087836:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.087845:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971910.087852:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971910.087858:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d200.
00000800:00000200:2.0:1788971910.087863:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971910.087871:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971910.087876:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971910.087888:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971910.087893:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0.
00000100:00000010:2.0:1788971910.087901:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f72a0.
00000800:00000200:2.0:1788971910.087995:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971910.087998:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d200.
00000400:00000200:2.0:1788971910.088011:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971910.088016:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971910.088028:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38
00000400:00000010:2.0:1788971910.088030:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38.
00000100:00000200:2.0:1788971910.088038:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c6a00 x1875872179069440/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971910.088890:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96a00.
00000400:00000200:3.0:1788971910.088901:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971910.088911:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78688.
00000400:00000010:3.0:1788971910.088916:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081000.
00000100:00000200:3.0:1788971910.088920:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069568, portal 10
00000400:00000010:3.0:1788971910.088927:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788971910.088929:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179069568, offset 0
00000400:00000200:3.0:1788971910.088934:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971910.088941:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971910.088946:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971910.088950:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971910.088953:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971910.088956:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971910.088958:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971910.088960:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [330:330:256:4294967295] 192.168.203.124@tcp LPNI seq info [330:330:8:4294967295]
00000400:00000200:3.0:1788971910.088964:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971910.088970:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971910.088975:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971910.088979:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00.
00000800:00000200:3.0:1788971910.088985:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971910.088990:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971910.088993:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971910.089054:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971910.089394:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971910.089398:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00.
00000400:00000200:2.0:1788971910.089402:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971910.089407:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971910.089412:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081088
00000400:00000010:2.0:1788971910.089414:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081088.
00000100:00000200:2.0:1788971910.089423:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c7480 x1875872179069568/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971926 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971910.092401:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.092410:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971910.092420:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.092424:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.092430:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.092434:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a200
00000400:00000200:2.0:1788971910.092439:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.092445:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x11d5 [1] + 224
00000400:00000200:2.0:1788971910.092462:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971910.092466:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971910.092479:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c6a00 x1875872179069440/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971926 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971910.095519:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b39f8.
00000400:00000200:3.0:1788971910.095530:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330
00000400:00000010:3.0:1788971910.095533:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330.
00000100:00000200:3.0:1788971910.095546:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c6a00 x1875872179069440/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971926 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971910.095557:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c6a00 x1875872179069440/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971926 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971910.095584:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69200.
02000000:00000010:3.0:1788971910.095603:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00.
00000100:00000010:3.0:1788971910.095608:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00.
00000400:00000200:2.0:1788971910.097154:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.097162:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971910.097172:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.097177:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.097182:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.097186:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a280
00000400:00000200:2.0:1788971910.097202:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971910.097209:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x11e5 [1] + 224
00000400:00000200:2.0:1788971910.097227:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971910.097232:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971910.097255:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c7480 x1875872179069568/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971926 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971910.097374:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78688.
00000400:00000200:3.0:1788971910.097379:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081000
00000400:00000010:3.0:1788971910.097381:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081000.
00000100:00000200:3.0:1788971910.097388:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c7480 x1875872179069568/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971926 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971910.097397:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c7480 x1875872179069568/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971926 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971910.097424:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96a00.
02000000:00000010:3.0:1788971910.097430:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400.
00000100:00000010:3.0:1788971910.097434:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480.
00000020:00000010:0.0:1788971910.206072:0:111133:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d947e0.
00000020:00000010:0.0:1788971910.206150:0:111133:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d947e0.
00000020:00000010:3.0:1788971910.228958:0:111134:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca950.
00000020:00000010:3.0:1788971910.229674:0:111134:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca950.
00000100:00000010:3.0:1788971910.792596:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f72a0.
00000100:00000010:2.0:1788971910.793365:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0.
00000100:00000010:3.0:1788971915.210425:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480.
02000000:00000010:3.0:1788971915.210451:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400.
00000100:00000010:3.0:1788971915.210494:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00.
02000000:00000010:3.0:1788971915.210500:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00.
02000000:00000010:3.0:1788971915.210533:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166be00.
00000400:00000200:3.0:1788971915.210542:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971915.210547:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78688.
00000400:00000010:3.0:1788971915.210553:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081000.
00000100:00000200:3.0:1788971915.210561:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069824, portal 10
02000000:00000010:2.0:1788971915.210573:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058e00.
00000400:00000010:3.0:1788971915.210582:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081660.
00000400:00000200:2.0:1788971915.210582:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000200:3.0:1788971915.210588:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179069824, offset 0
00000400:00000010:2.0:1788971915.210590:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3d68.
00000400:00000010:2.0:1788971915.210598:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38.
00000400:00000200:3.0:1788971915.210602:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000100:00000200:2.0:1788971915.210611:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069696, portal 25
00000400:00000200:3.0:1788971915.210612:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971915.210619:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0880.
00000400:00000200:3.0:1788971915.210623:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000100:00000200:2.0:1788971915.210623:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179069696, offset 0
00000400:00000200:3.0:1788971915.210632:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971915.210633:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971915.210639:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971915.210657:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971915.210660:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971915.210664:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [331:331:256:4294967295] 192.168.203.124@tcp LPNI seq info [331:331:8:4294967295]
00000400:00000200:3.0:1788971915.210669:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971915.210678:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971915.210685:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971915.210688:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000010:3.0:1788971915.210691:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00.
00000400:00000200:2.0:1788971915.210696:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971915.210701:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000800:00000200:3.0:1788971915.210702:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000400:00000200:2.0:1788971915.210706:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971915.210711:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000800:00000200:3.0:1788971915.210715:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000400:00000200:2.0:1788971915.210718:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000800:00000200:3.0:1788971915.210722:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971915.210724:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [332:332:255:4294967295] 192.168.203.124@tcp LPNI seq info [332:332:7:4294967295]
00000400:00000200:2.0:1788971915.210731:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971915.210739:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1788971915.210743:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971915.210748:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000100:00000010:3.0:1788971915.210749:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347a720.
00000800:00000010:2.0:1788971915.210753:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d200.
00000800:00000200:2.0:1788971915.210760:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000100:00000010:3.0:1788971915.210763:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347a540.
00000800:00000200:2.0:1788971915.210781:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971915.210786:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971915.210791:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971915.210830:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971915.210835:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00.
00000400:00000200:3.0:1788971915.210841:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971915.210846:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971915.210851:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081660
00000400:00000010:3.0:1788971915.210853:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081660.
00000100:00000200:3.0:1788971915.210863:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c6a00 x1875872179069824/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971915.210930:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971915.210932:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d200.
00000400:00000200:3.0:1788971915.210936:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971915.210939:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971915.210942:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0880
00000400:00000010:3.0:1788971915.210943:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0880.
00000100:00000200:3.0:1788971915.210948:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c7480 x1875872179069696/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971931 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971915.213430:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971915.213439:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971915.213449:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971915.213454:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971915.213459:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971915.213464:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a300
00000400:00000200:2.0:1788971915.213469:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971915.213475:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1205 [1] + 224
00000400:00000200:2.0:1788971915.213503:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971915.213508:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971915.213521:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c7480 x1875872179069696/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971931 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971915.213665:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3d68.
00000400:00000200:3.0:1788971915.213688:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38
00000400:00000010:3.0:1788971915.213691:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38.
00000100:00000200:3.0:1788971915.213704:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c7480 x1875872179069696/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971931 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971915.213717:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c7480 x1875872179069696/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971931 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971915.213761:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058e00.
02000000:00000010:3.0:1788971915.213767:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400.
00000100:00000010:3.0:1788971915.213773:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480.
00000400:00000200:3.0:1788971915.215033:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971915.215041:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971915.215050:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971915.215055:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971915.215061:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971915.215066:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a380
00000400:00000200:3.0:1788971915.215072:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971915.215078:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x11f5 [1] + 224
00000400:00000200:3.0:1788971915.215093:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971915.215098:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971915.215109:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c6a00 x1875872179069824/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971931 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971915.215230:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78688.
00000400:00000200:2.0:1788971915.215241:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081000
00000400:00000010:2.0:1788971915.215244:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081000.
00000100:00000200:2.0:1788971915.215363:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c6a00 x1875872179069824/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971931 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971915.215374:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c6a00 x1875872179069824/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971931 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971915.215407:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166be00.
02000000:00000010:2.0:1788971915.215414:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00.
00000100:00000010:2.0:1788971915.215419:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00.
00000100:00000010:3.0:1788971915.911504:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347a540.
00000100:00000010:2.0:1788971915.913448:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347a720.
00000100:00000010:0.0:1788971920.327489:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700.
02000000:00000010:0.0:1788971920.327512:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971920.327555:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971920.327562:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400.
02000000:00000010:0.0:1788971920.327596:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017a00.
00000400:00000200:0.0:1788971920.327606:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971920.327610:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163630.
00000400:00000010:0.0:1788971920.327616:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480844c8.
00000100:00000200:0.0:1788971920.327622:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070080, portal 10
00000400:00000010:0.0:1788971920.327629:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084a18.
00000100:00000200:0.0:1788971920.327632:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179070080, offset 0
00000400:00000200:0.0:1788971920.327641:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971920.327649:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971920.327658:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971920.327663:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971920.327668:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971920.327671:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971920.327674:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971920.327678:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [333:333:256:4294967295] 192.168.203.124@tcp LPNI seq info [333:333:8:4294967295]
00000400:00000200:0.0:1788971920.327683:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971920.327692:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971920.327699:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971920.327703:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500.
00000800:00000200:0.0:1788971920.327710:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971920.327717:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971920.327721:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971920.327746:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971920.327752:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969ae0.
00000100:00000010:0.0:1788971920.327766:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969c60.
02000000:00000010:0.0:1788971920.327794:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017200.
00000400:00000200:0.0:1788971920.327810:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971920.327813:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163210.
00000800:00000200:2.0:1788971920.327864:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971920.327870:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500.
00000400:00000200:2.0:1788971920.327876:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971920.327881:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971920.327887:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084a18
00000400:00000010:2.0:1788971920.327889:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084a18.
00000100:00000200:2.0:1788971920.327898:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179070080/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971920.327987:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480846e8.
00000100:00000200:0.0:1788971920.327992:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179069952, portal 25
00000400:00000010:0.0:1788971920.327997:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084b28.
00000100:00000200:0.0:1788971920.327999:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179069952, offset 0
00000400:00000200:0.0:1788971920.328004:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971920.328009:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971920.328012:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971920.328015:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971920.328017:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971920.328020:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971920.328022:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971920.328025:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [334:334:256:4294967295] 192.168.203.124@tcp LPNI seq info [334:334:8:4294967295]
00000400:00000200:0.0:1788971920.328028:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971920.328035:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971920.328039:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971920.328043:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800.
00000800:00000200:0.0:1788971920.328047:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971920.328051:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971920.328054:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971920.328109:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971920.330477:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971920.330486:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800.
00000400:00000200:3.0:1788971920.330492:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971920.330499:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971920.330504:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084b28
00000400:00000010:3.0:1788971920.330507:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084b28.
00000100:00000200:3.0:1788971920.330515:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c700 x1875872179069952/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971936 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971920.331904:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971920.331912:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971920.331919:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971920.331923:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971920.331928:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971920.331932:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a480
00000400:00000200:2.0:1788971920.331937:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971920.331943:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1215 [1] + 224
00000400:00000200:2.0:1788971920.331956:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971920.331961:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971920.331971:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179070080/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971936 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971920.332062:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163630.
00000400:00000200:0.0:1788971920.332068:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480844c8
00000400:00000010:0.0:1788971920.332071:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480844c8.
00000100:00000200:0.0:1788971920.332078:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179070080/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971936 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971920.332088:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179070080/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971936 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971920.332120:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017a00.
02000000:00000010:0.0:1788971920.332136:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400.
00000100:00000010:0.0:1788971920.332140:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000400:00000200:3.0:1788971920.333373:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971920.333380:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971920.333387:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971920.333390:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971920.333394:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971920.333398:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a400
00000400:00000200:3.0:1788971920.333403:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971920.333409:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x121d [1] + 224
00000400:00000200:3.0:1788971920.333419:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971920.333423:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971920.333432:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c700 x1875872179069952/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971936 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971920.333521:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163210.
00000400:00000200:0.0:1788971920.333526:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480846e8
00000400:00000010:0.0:1788971920.333528:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480846e8.
00000100:00000200:0.0:1788971920.333539:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c700 x1875872179069952/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971936 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971920.333548:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c700 x1875872179069952/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971936 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971920.333565:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017200.
02000000:00000010:0.0:1788971920.333569:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971920.333573:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700.
00000100:00000010:2.0:1788971921.031423:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969ae0.
00000100:00000010:3.0:1788971921.032008:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969c60.
00000100:00000010:2.0:1788971925.447686:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411180.
02000000:00000010:2.0:1788971925.447711:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381d800.
00000100:00000010:2.0:1788971925.447946:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412300.
02000000:00000010:2.0:1788971925.447953:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381dd00.
02000000:00000010:2.0:1788971925.447986:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058600.
00000400:00000200:2.0:1788971925.447995:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971925.447998:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3108.
00000400:00000010:2.0:1788971925.448005:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788971925.448012:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070336, portal 10
00000400:00000010:2.0:1788971925.448019:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971925.448021:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179070336, offset 0
00000400:00000200:2.0:1788971925.448032:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971925.448052:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971925.448060:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971925.448065:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971925.448069:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971925.448087:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971925.448090:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971925.448093:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [335:335:256:4294967295] 192.168.203.124@tcp LPNI seq info [335:335:8:4294967295]
00000400:00000200:2.0:1788971925.448098:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971925.448109:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971925.448115:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971925.448120:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d200.
00000800:00000200:2.0:1788971925.448127:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971925.448145:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971925.448150:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971925.448166:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971925.448171:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0.
00000100:00000010:2.0:1788971925.448181:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7780.
02000000:00000010:2.0:1788971925.448205:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058200.
00000400:00000200:2.0:1788971925.448210:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971925.448212:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b34d0.
00000400:00000010:2.0:1788971925.448216:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0.
00000100:00000200:2.0:1788971925.448219:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070208, portal 25
00000400:00000010:2.0:1788971925.448223:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8.
00000100:00000200:2.0:1788971925.448224:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179070208, offset 0
00000400:00000200:2.0:1788971925.448229:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971925.448245:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971925.448249:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971925.448251:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971925.448253:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971925.448256:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971925.448258:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971925.448261:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [336:336:255:4294967295] 192.168.203.124@tcp LPNI seq info [336:336:7:4294967295]
00000400:00000200:2.0:1788971925.448264:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971925.448270:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971925.448283:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971925.448286:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d000.
00000800:00000200:2.0:1788971925.448291:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971925.448295:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971925.448297:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971925.448338:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971925.448391:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971925.448395:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d200.
00000400:00000200:2.0:1788971925.448401:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971925.448406:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971925.448411:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0
00000400:00000010:2.0:1788971925.448413:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971925.448422:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412300 x1875872179070336/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971925.448498:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971925.448501:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d000.
00000400:00000200:2.0:1788971925.448505:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971925.448508:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971925.448511:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8
00000400:00000010:2.0:1788971925.448524:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8.
00000100:00000200:2.0:1788971925.448529:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43411180 x1875872179070208/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971941 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971925.452545:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452558:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971925.452600:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452606:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452613:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452617:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a500
00000400:00000200:3.0:1788971925.452623:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452631:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1235 [1] + 224
00000400:00000200:3.0:1788971925.452646:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971925.452651:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971925.452670:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43411180 x1875872179070208/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971941 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971925.452715:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452719:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971925.452724:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452745:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452748:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452751:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a580
00000400:00000200:3.0:1788971925.452756:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971925.452760:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1225 [1] + 224
00000400:00000200:3.0:1788971925.452770:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971925.452774:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971925.452782:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412300 x1875872179070336/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971941 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971925.453639:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b34d0.
00000400:00000200:2.0:1788971925.453650:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0
00000400:00000010:2.0:1788971925.453653:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0.
00000100:00000200:2.0:1788971925.453664:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43411180 x1875872179070208/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971941 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971925.453675:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43411180 x1875872179070208/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971941 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971925.453710:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058200.
02000000:00000010:2.0:1788971925.453730:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381d800.
00000100:00000010:2.0:1788971925.453737:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411180.
00000400:00000010:2.0:1788971925.454961:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3108.
00000400:00000200:2.0:1788971925.454968:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8
00000400:00000010:2.0:1788971925.454971:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788971925.454984:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412300 x1875872179070336/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971941 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971925.454995:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412300 x1875872179070336/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971941 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971925.455024:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058600.
02000000:00000010:2.0:1788971925.455030:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381dd00.
00000100:00000010:2.0:1788971925.455034:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412300.
00000100:00000010:2.0:1788971926.151533:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0.
00000100:00000010:3.0:1788971926.152088:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7780.
00000100:00000010:0.0:1788971930.567494:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700.
02000000:00000010:0.0:1788971930.567523:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971930.567566:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971930.567571:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400.
02000000:00000010:0.0:1788971930.567605:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017a00.
00000400:00000200:0.0:1788971930.567616:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971930.567620:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163210.
00000400:00000010:0.0:1788971930.567627:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480846e8.
00000100:00000200:0.0:1788971930.567633:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070592, portal 10
00000400:00000010:0.0:1788971930.567640:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480844c8.
00000100:00000200:0.0:1788971930.567643:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179070592, offset 0
00000400:00000200:0.0:1788971930.567652:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971930.567673:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971930.567680:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971930.567685:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971930.567689:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971930.567692:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971930.567695:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971930.567699:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [337:337:256:4294967295] 192.168.203.124@tcp LPNI seq info [337:337:8:4294967295]
00000400:00000200:0.0:1788971930.567726:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971930.567736:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971930.567742:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971930.567746:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55a00.
00000800:00000200:0.0:1788971930.567752:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971930.567758:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971930.567766:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971930.567787:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971930.567792:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969240.
00000100:00000010:0.0:1788971930.567800:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449693c0.
02000000:00000010:0.0:1788971930.567831:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016800.
00000400:00000200:0.0:1788971930.567836:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971930.567839:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163630.
00000400:00000010:0.0:1788971930.567842:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084dd0.
00000100:00000200:0.0:1788971930.567846:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070464, portal 25
00000400:00000010:0.0:1788971930.567849:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084c38.
00000100:00000200:0.0:1788971930.567851:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179070464, offset 0
00000400:00000200:0.0:1788971930.567856:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971930.567859:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971930.567863:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971930.567865:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971930.567868:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971930.567870:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971930.567872:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971930.567875:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [338:338:255:4294967295] 192.168.203.124@tcp LPNI seq info [338:338:7:4294967295]
00000400:00000200:0.0:1788971930.567879:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971930.567884:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971930.568150:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971930.568154:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55a00.
00000400:00000200:2.0:1788971930.568160:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971930.568166:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971930.568172:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480844c8
00000400:00000010:2.0:1788971930.568174:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480844c8.
00000100:00000200:2.0:1788971930.568183:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179070592/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971930.569810:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.569818:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971930.569825:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.569830:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.569835:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.569839:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a680
00000400:00000200:3.0:1788971930.569843:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.569849:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1245 [1] + 224
00000400:00000200:3.0:1788971930.569860:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971930.569865:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971930.569875:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179070592/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971946 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1788971930.570000:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971930.570006:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00.
00000800:00000200:0.0:1788971930.570012:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971930.570030:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971930.570034:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971930.570109:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971930.570126:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163210.
00000400:00000200:0.0:1788971930.570131:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480846e8
00000400:00000010:0.0:1788971930.570133:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480846e8.
00000100:00000200:0.0:1788971930.570140:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179070592/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971946 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971930.570150:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179070592/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971946 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971930.570179:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017a00.
02000000:00000010:0.0:1788971930.570185:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400.
00000100:00000010:0.0:1788971930.570190:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000800:00000200:2.0:1788971930.570424:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971930.570429:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00.
00000400:00000200:2.0:1788971930.570432:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971930.570436:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971930.570440:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084c38
00000400:00000010:2.0:1788971930.570442:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084c38.
00000100:00000200:2.0:1788971930.570448:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c700 x1875872179070464/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971946 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971930.572015:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.572026:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971930.572036:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.572042:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.572047:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.572052:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a600
00000400:00000200:3.0:1788971930.572058:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971930.572081:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x124d [1] + 224
00000400:00000200:3.0:1788971930.572110:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971930.572115:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971930.572130:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c700 x1875872179070464/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971946 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971930.572793:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163630.
00000400:00000200:0.0:1788971930.572802:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084dd0
00000400:00000010:0.0:1788971930.572805:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084dd0.
00000100:00000200:0.0:1788971930.572814:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c700 x1875872179070464/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971946 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971930.572825:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c700 x1875872179070464/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971946 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971930.572858:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016800.
02000000:00000010:0.0:1788971930.572866:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971930.572871:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700.
00000100:00000010:2.0:1788971931.271693:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969240.
00000100:00000010:3.0:1788971931.272944:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449693c0.
00000100:00000010:3.0:1788971935.687792:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480.
02000000:00000010:3.0:1788971935.687827:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400.
02000000:00000010:2.0:1788971935.688012:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058600.
00000400:00000200:2.0:1788971935.688054:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971935.688060:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3108.
00000400:00000010:2.0:1788971935.688078:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788971935.688087:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070720, portal 25
00000400:00000010:2.0:1788971935.688095:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0.
00000100:00000200:2.0:1788971935.688098:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179070720, offset 0
00000400:00000200:2.0:1788971935.688109:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971935.688120:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.688130:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971935.688137:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971935.688142:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971935.688146:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971935.688150:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971935.688155:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [339:339:256:4294967295] 192.168.203.124@tcp LPNI seq info [339:339:8:4294967295]
00000400:00000200:2.0:1788971935.688160:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.688187:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971935.688195:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971935.688201:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d000.
00000800:00000200:2.0:1788971935.688210:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971935.688220:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971935.688226:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971935.688245:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971935.688252:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7ea0.
00000100:00000010:2.0:1788971935.688265:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f71e0.
00000100:00000010:3.0:1788971935.688385:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180.
02000000:00000010:3.0:1788971935.688394:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00.
02000000:00000010:2.0:1788971935.689037:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058200.
00000400:00000200:2.0:1788971935.689048:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971935.689053:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b34d0.
00000400:00000010:2.0:1788971935.689059:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8.
00000100:00000200:2.0:1788971935.689073:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070848, portal 10
00000400:00000010:2.0:1788971935.689081:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971935.689084:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179070848, offset 0
00000400:00000200:2.0:1788971935.689106:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971935.689113:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.689119:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971935.689124:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971935.689128:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971935.689131:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971935.689134:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971935.689137:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [340:340:255:4294967295] 192.168.203.124@tcp LPNI seq info [340:340:7:4294967295]
00000400:00000200:2.0:1788971935.689141:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.689149:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971935.689155:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971935.689159:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700.
00000800:00000200:2.0:1788971935.689165:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971935.689579:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971935.689584:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971935.689595:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971935.689675:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971935.689680:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d000.
00000400:00000200:2.0:1788971935.689688:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971935.689692:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971935.689699:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0
00000400:00000010:2.0:1788971935.689702:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0.
00000100:00000200:2.0:1788971935.689714:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c7480 x1875872179070720/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971935.689809:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971935.689812:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700.
00000400:00000200:2.0:1788971935.689815:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971935.689819:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971935.689822:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0
00000400:00000010:2.0:1788971935.689824:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971935.689829:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c5180 x1875872179070848/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971951 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971935.695392:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971935.695405:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971935.695415:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971935.695421:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971935.695433:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971935.695438:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a780
00000400:00000200:3.0:1788971935.695445:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971935.695452:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1265 [1] + 224
00000400:00000200:3.0:1788971935.695469:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971935.695475:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971935.695494:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c5180 x1875872179070848/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971951 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971935.695705:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b34d0.
00000400:00000200:2.0:1788971935.695711:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8
00000400:00000010:2.0:1788971935.695715:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8.
00000100:00000200:2.0:1788971935.695723:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c5180 x1875872179070848/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971951 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971935.695733:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c5180 x1875872179070848/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971951 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971935.695776:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058200.
02000000:00000010:2.0:1788971935.695790:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00.
00000100:00000010:2.0:1788971935.695795:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180.
00000400:00000200:2.0:1788971935.702659:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.702673:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971935.702687:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.702693:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.702700:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.702705:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a700
00000400:00000200:2.0:1788971935.702712:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971935.702720:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1255 [1] + 224
00000400:00000200:2.0:1788971935.702756:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971935.702762:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971935.702780:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c7480 x1875872179070720/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971951 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971935.703010:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3108.
00000400:00000200:3.0:1788971935.703017:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8
00000400:00000010:3.0:1788971935.703020:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8.
00000100:00000200:3.0:1788971935.703038:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c7480 x1875872179070720/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971951 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971935.703048:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c7480 x1875872179070720/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971951 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971935.703114:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058600.
02000000:00000010:3.0:1788971935.703122:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400.
00000100:00000010:3.0:1788971935.703128:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480.
00000100:00000010:3.0:1788971936.392711:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f71e0.
00000100:00000010:2.0:1788971936.394020:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7ea0.
00000100:00000010:0.0:1788971940.812393:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700.
02000000:00000010:0.0:1788971940.812415:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971940.812460:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971940.812465:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400.
02000000:00000010:0.0:1788971940.812893:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016800.
00000400:00000200:0.0:1788971940.812904:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971940.812908:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163630.
00000400:00000010:0.0:1788971940.812928:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084dd0.
00000100:00000200:0.0:1788971940.812934:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071104, portal 10
00000400:00000010:0.0:1788971940.812942:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480846e8.
00000100:00000200:0.0:1788971940.812945:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179071104, offset 0
00000400:00000200:0.0:1788971940.812954:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971940.812963:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971940.812971:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971940.812977:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971940.812981:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971940.812985:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971940.812988:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971940.812991:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [341:341:256:4294967295] 192.168.203.124@tcp LPNI seq info [341:341:8:4294967295]
00000400:00000200:0.0:1788971940.812995:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971940.813005:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971940.813011:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971940.813016:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000.
00000800:00000200:0.0:1788971940.813023:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971940.813041:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971940.813046:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971940.813066:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971940.813071:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449697e0.
00000100:00000010:0.0:1788971940.813080:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969420.
02000000:00000010:0.0:1788971940.813104:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017a00.
00000400:00000200:0.0:1788971940.813108:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971940.813111:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163210.
00000400:00000010:0.0:1788971940.813115:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084880.
00000100:00000200:0.0:1788971940.813117:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179070976, portal 25
00000400:00000010:0.0:1788971940.813122:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084c38.
00000100:00000200:0.0:1788971940.813124:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179070976, offset 0
00000400:00000200:0.0:1788971940.813128:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971940.813132:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971940.813141:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971940.813144:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971940.813147:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971940.813149:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971940.813151:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971940.813154:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [342:342:255:4294967295] 192.168.203.124@tcp LPNI seq info [342:342:7:4294967295]
00000400:00000200:0.0:1788971940.813157:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971940.813163:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971940.813168:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971940.813172:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00.
00000800:00000200:0.0:1788971940.813176:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971940.813180:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971940.813183:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971940.813187:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971940.813962:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971940.813971:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000.
00000400:00000200:3.0:1788971940.813980:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971940.813987:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971940.813993:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480846e8
00000400:00000010:3.0:1788971940.813996:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480846e8.
00000100:00000200:3.0:1788971940.814005:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179071104/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971940.814095:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971940.814097:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00.
00000400:00000200:3.0:1788971940.814101:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971940.814104:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971940.814107:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084c38
00000400:00000010:3.0:1788971940.814109:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084c38.
00000100:00000200:3.0:1788971940.814114:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c700 x1875872179070976/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971956 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971940.819677:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819686:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971940.819694:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819699:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819704:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819824:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a880
00000400:00000200:2.0:1788971940.819830:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819836:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1275 [1] + 224
00000400:00000200:2.0:1788971940.819848:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971940.819852:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971940.819866:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179071104/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971956 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971940.819906:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819921:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971940.819926:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819930:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819934:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971940.819937:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a800
00000400:00000200:2.0:1788971940.819941:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971940.819944:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163630.
00000400:00000200:0.0:1788971940.819950:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084dd0
00000400:00000010:0.0:1788971940.819953:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084dd0.
00000400:00000200:2.0:1788971940.819962:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x127d [1] + 224
00000100:00000200:0.0:1788971940.819963:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179071104/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971956 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971940.819977:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000100:00000200:0.0:1788971940.819978:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179071104/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971956 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971940.819985:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971940.819996:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c700 x1875872179070976/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971956 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971940.820019:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016800.
02000000:00000010:0.0:1788971940.820392:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400.
00000100:00000010:0.0:1788971940.820398:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000400:00000010:0.0:1788971940.820406:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163210.
00000400:00000200:0.0:1788971940.820409:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084880
00000400:00000010:0.0:1788971940.820411:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084880.
00000100:00000200:0.0:1788971940.820417:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c700 x1875872179070976/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971956 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971940.820426:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c700 x1875872179070976/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971956 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971940.820449:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017a00.
02000000:00000010:0.0:1788971940.820453:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971940.820455:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700.
00000100:00000010:3.0:1788971941.511632:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969420.
00000100:00000010:2.0:1788971941.511652:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449697e0.
00000100:00000010:2.0:1788971945.928060:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412300.
02000000:00000010:2.0:1788971945.928081:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700.
00000100:00000010:2.0:1788971945.928118:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411180.
02000000:00000010:2.0:1788971945.928122:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d000.
02000000:00000010:2.0:1788971945.928157:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058200.
00000400:00000200:2.0:1788971945.928167:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971945.928171:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b34d0.
00000400:00000010:2.0:1788971945.928177:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8.
00000100:00000200:2.0:1788971945.928184:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071360, portal 10
00000400:00000010:2.0:1788971945.928190:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971945.928208:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179071360, offset 0
02000000:00000010:3.0:1788971945.928322:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166bc00.
00000400:00000200:3.0:1788971945.928328:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971945.928330:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78bb0.
00000400:00000200:2.0:1788971945.928413:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971945.928423:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971945.928440:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971945.928445:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971945.928449:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971945.928452:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971945.928455:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971945.928459:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [343:343:256:4294967295] 192.168.203.124@tcp LPNI seq info [343:343:8:4294967295]
00000400:00000200:2.0:1788971945.928463:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971945.928471:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971945.928477:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971945.928482:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0da00.
00000800:00000200:2.0:1788971945.928488:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971945.928496:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971945.928500:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0da00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971945.928526:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971945.928532:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300.
00000100:00000010:2.0:1788971945.928549:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840.
00000800:00000200:2.0:1788971945.928652:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971945.928656:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0da00.
00000400:00000200:2.0:1788971945.928659:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971945.928664:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000010:3.0:1788971945.931964:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081660.
00000100:00000200:3.0:1788971945.931979:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071232, portal 25
00000400:00000010:3.0:1788971945.931986:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081550.
00000100:00000200:3.0:1788971945.932000:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179071232, offset 0
00000400:00000200:3.0:1788971945.932012:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971945.932022:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932046:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971945.932051:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971945.932055:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971945.932059:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971945.932061:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971945.932065:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [344:344:255:4294967295] 192.168.203.124@tcp LPNI seq info [344:344:7:4294967295]
00000400:00000200:3.0:1788971945.932069:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932120:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971945.932127:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971945.932143:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d400.
00000800:00000200:3.0:1788971945.932150:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971945.932158:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971945.932162:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971945.932169:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932250:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932255:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971945.932261:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932264:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932268:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932271:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0a980
00000400:00000200:3.0:1788971945.932276:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.932281:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1285 [1] + 224
00000400:00000200:3.0:1788971945.932293:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971945.932298:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971945.932328:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43411180 x1875872179071360/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971945.932365:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b34d0.
00000400:00000200:3.0:1788971945.932369:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8
00000400:00000010:3.0:1788971945.932372:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8.
00000100:00000200:3.0:1788971945.932377:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43411180 x1875872179071360/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971961 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971945.932394:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43411180 x1875872179071360/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971961 ref 2 fl Rpc:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971945.935345:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0
00000400:00000010:2.0:1788971945.935355:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971945.935374:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43411180 x1875872179071360/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971961 ref 2 fl UnregRPC:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971945.935504:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971945.935508:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d400.
00000400:00000200:2.0:1788971945.935512:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971945.935518:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971945.935535:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081550
00000400:00000010:2.0:1788971945.935537:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081550.
00000100:00000200:2.0:1788971945.935555:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412300 x1875872179071232/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971961 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971945.935731:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058200.
02000000:00000010:3.0:1788971945.935739:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d000.
00000100:00000010:3.0:1788971945.935743:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411180.
00000400:00000200:3.0:1788971945.940133:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.940144:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971945.940153:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.940157:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.940161:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.940166:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0a900
00000400:00000200:3.0:1788971945.940171:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971945.940178:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1295 [1] + 224
00000400:00000200:3.0:1788971945.940192:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971945.940197:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971945.940213:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412300 x1875872179071232/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971961 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971945.940347:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78bb0.
00000400:00000200:2.0:1788971945.940487:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081660
00000400:00000010:2.0:1788971945.940491:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081660.
00000100:00000200:2.0:1788971945.940499:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412300 x1875872179071232/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971961 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971945.940509:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412300 x1875872179071232/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971961 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971945.940536:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166bc00.
02000000:00000010:2.0:1788971945.940541:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700.
00000100:00000010:2.0:1788971945.940546:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412300.
00000100:00000010:3.0:1788971946.631939:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840.
00000100:00000010:2.0:1788971946.633232:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300.
00000100:00000010:2.0:1788971951.047574:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412300.
02000000:00000010:2.0:1788971951.047605:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700.
00000100:00000010:2.0:1788971951.047652:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971951.047658:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0da00.
02000000:00000010:3.0:1788971951.047917:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a600.
00000400:00000200:3.0:1788971951.047930:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971951.047935:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e782c0.
00000400:00000010:3.0:1788971951.047944:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788971951.047962:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071616, portal 10
00000400:00000010:3.0:1788971951.047971:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908.
00000100:00000200:3.0:1788971951.047974:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179071616, offset 0
00000400:00000200:3.0:1788971951.047985:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971951.047996:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.048005:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971951.048011:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971951.048032:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971951.048036:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971951.048039:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971951.048043:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [345:345:256:4294967295] 192.168.203.124@tcp LPNI seq info [345:345:8:4294967295]
00000400:00000200:3.0:1788971951.048047:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.048057:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971951.048064:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971951.048070:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d700.
00000800:00000200:3.0:1788971951.048078:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971951.048087:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971951.048093:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971951.048113:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971951.048119:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347a900.
00000100:00000010:3.0:1788971951.048135:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347ad80.
02000000:00000010:3.0:1788971951.048164:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166bc00.
00000400:00000200:3.0:1788971951.048170:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971951.048173:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78058.
00000400:00000010:3.0:1788971951.048176:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081e58.
00000100:00000200:3.0:1788971951.048179:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071488, portal 25
00000400:00000010:3.0:1788971951.048183:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788971951.048185:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179071488, offset 0
00000400:00000200:3.0:1788971951.048189:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971951.048193:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.048196:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971951.048199:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971951.048201:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971951.048203:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971951.048205:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971951.048207:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [346:346:255:4294967295] 192.168.203.124@tcp LPNI seq info [346:346:7:4294967295]
00000400:00000200:3.0:1788971951.048211:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.048216:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971951.048230:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971951.048233:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200.
00000800:00000200:3.0:1788971951.048237:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971951.048242:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971951.048245:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971951.048250:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971951.049063:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971951.049072:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d700.
00000400:00000200:2.0:1788971951.049081:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971951.049088:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971951.049095:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908
00000400:00000010:2.0:1788971951.049098:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908.
00000100:00000200:2.0:1788971951.049110:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179071616/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971951.049612:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971951.049618:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200.
00000400:00000200:2.0:1788971951.049622:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971951.049627:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971951.049643:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480815d8
00000400:00000010:2.0:1788971951.049646:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480815d8.
00000100:00000200:2.0:1788971951.049653:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412300 x1875872179071488/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971967 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971951.057280:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057293:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971951.057405:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057411:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057417:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057422:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0aa00
00000400:00000200:3.0:1788971951.057428:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057436:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x12b5 [1] + 224
00000400:00000200:3.0:1788971951.057451:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971951.057456:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971951.057473:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412300 x1875872179071488/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971967 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971951.057582:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057586:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971951.057590:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057593:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057596:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057599:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0aa80
00000400:00000200:3.0:1788971951.057613:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971951.057618:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x12a5 [1] + 224
00000400:00000200:3.0:1788971951.057628:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971951.057631:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971951.057637:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179071616/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971967 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971951.059007:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78058.
00000400:00000200:2.0:1788971951.059021:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081e58
00000400:00000010:2.0:1788971951.059024:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081e58.
00000100:00000200:2.0:1788971951.059038:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412300 x1875872179071488/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971967 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971951.059050:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412300 x1875872179071488/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971967 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971951.059081:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e782c0.
00000400:00000200:2.0:1788971951.059084:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081220
00000400:00000010:2.0:1788971951.059086:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081220.
00000100:00000200:2.0:1788971951.059091:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179071616/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971967 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971951.059099:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179071616/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971967 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971951.059124:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166bc00.
02000000:00000010:2.0:1788971951.059132:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700.
00000100:00000010:2.0:1788971951.059137:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412300.
02000000:00000010:2.0:1788971951.059141:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a600.
02000000:00000010:2.0:1788971951.059144:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0da00.
00000100:00000010:2.0:1788971951.059146:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
00000100:00000010:2.0:1788971951.751474:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347a900.
00000100:00000010:3.0:1788971951.751594:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347ad80.
00000100:00000010:3.0:1788971956.168108:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480.
02000000:00000010:3.0:1788971956.168133:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dc00.
00000100:00000010:3.0:1788971956.168172:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4000.
02000000:00000010:3.0:1788971956.168179:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200.
02000000:00000010:2.0:1788971956.168679:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54059800.
00000400:00000200:2.0:1788971956.168691:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971956.168695:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160.
00000400:00000010:2.0:1788971956.168701:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971956.168708:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071744, portal 25
00000400:00000010:2.0:1788971956.168724:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0.
00000100:00000200:2.0:1788971956.168727:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179071744, offset 0
00000400:00000200:2.0:1788971956.168736:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971956.168744:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971956.168751:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971956.168756:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971956.168760:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971956.168764:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971956.168767:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971956.168770:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [347:347:256:4294967295] 192.168.203.124@tcp LPNI seq info [347:347:8:4294967295]
00000400:00000200:2.0:1788971956.168774:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971956.168784:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971956.168789:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971956.168794:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0da00.
00000800:00000200:2.0:1788971956.168800:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971956.168807:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971956.168812:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0da00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971956.168833:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971956.168838:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7c00.
00000100:00000010:2.0:1788971956.168848:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0.
02000000:00000010:2.0:1788971956.168870:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058800.
00000400:00000200:2.0:1788971956.168874:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971956.168877:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3420.
00000400:00000010:2.0:1788971956.168881:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8.
00000100:00000200:2.0:1788971956.168883:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179071872, portal 10
00000400:00000010:2.0:1788971956.168888:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28.
00000100:00000200:2.0:1788971956.168890:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179071872, offset 0
00000400:00000200:2.0:1788971956.168895:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971956.168898:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971956.168901:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971956.168904:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971956.168913:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971956.168926:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971956.168929:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971956.168931:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [348:348:255:4294967295] 192.168.203.124@tcp LPNI seq info [348:348:7:4294967295]
00000400:00000200:2.0:1788971956.168936:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971956.168941:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971956.168945:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971956.168948:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700.
00000800:00000200:2.0:1788971956.168952:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971956.168956:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971956.168958:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971956.168963:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971956.169186:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971956.169191:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0da00.
00000400:00000200:3.0:1788971956.169198:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971956.169203:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971956.169209:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0
00000400:00000010:3.0:1788971956.169212:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0.
00000100:00000200:3.0:1788971956.169221:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c7480 x1875872179071744/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971956.169584:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971956.169587:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700.
00000400:00000200:3.0:1788971956.169590:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971956.169595:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971956.169598:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28
00000400:00000010:3.0:1788971956.169600:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28.
00000100:00000200:3.0:1788971956.169605:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c4000 x1875872179071872/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971972 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971956.173537:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173546:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971956.173554:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173559:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173563:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173567:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ab80
00000400:00000200:3.0:1788971956.173572:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173578:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x12d5 [1] + 224
00000400:00000200:3.0:1788971956.173588:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971956.173606:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971956.173620:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c4000 x1875872179071872/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971972 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971956.173655:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173658:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971956.173661:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173663:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173666:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173668:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ab00
00000400:00000200:3.0:1788971956.173672:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971956.173680:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x12c5 [1] + 224
00000400:00000200:3.0:1788971956.173687:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971956.173690:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971956.173695:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c7480 x1875872179071744/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971972 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971956.174687:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3420.
00000400:00000200:2.0:1788971956.174695:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8
00000400:00000010:2.0:1788971956.174697:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8.
00000100:00000200:2.0:1788971956.174705:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c4000 x1875872179071872/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971972 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971956.174715:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c4000 x1875872179071872/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971972 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971956.174745:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160.
00000400:00000200:2.0:1788971956.174748:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0
00000400:00000010:2.0:1788971956.174750:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971956.174754:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c7480 x1875872179071744/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971972 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971956.174763:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c7480 x1875872179071744/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971972 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971956.174778:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058800.
02000000:00000010:2.0:1788971956.174784:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200.
00000100:00000010:2.0:1788971956.174790:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4000.
02000000:00000010:2.0:1788971956.174807:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54059800.
02000000:00000010:2.0:1788971956.174810:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dc00.
00000100:00000010:2.0:1788971956.174812:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480.
00000100:00000010:2.0:1788971956.871510:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7c00.
00000100:00000010:3.0:1788971956.871704:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0.
00000100:00000010:3.0:1788971961.287457:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6300.
02000000:00000010:3.0:1788971961.287479:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700.
00000100:00000010:3.0:1788971961.287527:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880.
02000000:00000010:3.0:1788971961.287532:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400.
02000000:00000010:3.0:1788971961.287568:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166bc00.
00000400:00000200:3.0:1788971961.287587:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971961.287592:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e786e0.
00000400:00000010:3.0:1788971961.287598:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081198.
00000100:00000200:3.0:1788971961.287605:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072128, portal 10
00000400:00000010:3.0:1788971961.287614:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788971961.287617:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179072128, offset 0
00000400:00000200:3.0:1788971961.287626:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971961.287634:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.287641:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971961.287662:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971961.287667:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971961.287671:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971961.287674:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971961.287678:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [349:349:256:4294967295] 192.168.203.124@tcp LPNI seq info [349:349:8:4294967295]
00000400:00000200:3.0:1788971961.287682:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.287692:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971961.287698:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971961.287702:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00.
00000800:00000200:3.0:1788971961.287709:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971961.287716:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971961.287721:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971961.287737:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788971961.287742:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347ad80.
00000100:00000010:3.0:1788971961.287756:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4347a600.
02000000:00000010:3.0:1788971961.287776:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166be00.
00000400:00000200:3.0:1788971961.287780:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971961.287782:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78898.
00000400:00000010:3.0:1788971961.287786:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081e58.
00000100:00000200:3.0:1788971961.287789:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072000, portal 25
00000400:00000010:3.0:1788971961.287792:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788971961.287794:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179072000, offset 0
00000400:00000200:3.0:1788971961.287798:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788971961.287802:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.287805:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971961.287808:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971961.287810:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971961.287812:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971961.287814:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971961.287817:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [350:350:255:4294967295] 192.168.203.124@tcp LPNI seq info [350:350:7:4294967295]
00000400:00000200:3.0:1788971961.287820:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.287832:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971961.287837:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971961.287840:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00.
00000800:00000200:3.0:1788971961.287843:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971961.287847:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971961.287850:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971961.287854:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971961.287907:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971961.288229:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00.
00000400:00000200:3.0:1788971961.288235:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971961.288239:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971961.288244:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081220
00000400:00000010:3.0:1788971961.288246:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788971961.288255:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c5880 x1875872179072128/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971961.291318:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971961.291327:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00.
00000400:00000200:3.0:1788971961.291333:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971961.291346:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971961.291353:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480815d8
00000400:00000010:3.0:1788971961.291356:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788971961.291368:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c6300 x1875872179072000/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971977 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971961.292869:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.292891:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971961.292903:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.292909:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.292916:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.292920:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ac80
00000400:00000200:3.0:1788971961.292927:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.292936:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x12e5 [1] + 224
00000400:00000200:3.0:1788971961.292987:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971961.292993:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971961.293011:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c5880 x1875872179072128/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971977 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971961.293166:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e786e0.
00000400:00000200:2.0:1788971961.293173:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081198
00000400:00000010:2.0:1788971961.293177:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081198.
00000100:00000200:2.0:1788971961.293185:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c5880 x1875872179072128/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971977 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971961.293194:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c5880 x1875872179072128/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971977 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971961.293238:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166bc00.
02000000:00000010:2.0:1788971961.293257:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400.
00000100:00000010:2.0:1788971961.293263:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880.
00000400:00000200:3.0:1788971961.295040:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.295051:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971961.295061:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.295066:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.295071:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.295076:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ac00
00000400:00000200:3.0:1788971961.295082:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971961.295089:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x12f5 [1] + 224
00000400:00000200:3.0:1788971961.295105:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971961.295110:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971961.295126:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c6300 x1875872179072000/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971977 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971961.295705:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78898.
00000400:00000200:2.0:1788971961.295715:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081e58
00000400:00000010:2.0:1788971961.295717:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081e58.
00000100:00000200:2.0:1788971961.295726:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c6300 x1875872179072000/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971977 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971961.295737:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c6300 x1875872179072000/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971977 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971961.295770:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166be00.
02000000:00000010:2.0:1788971961.295777:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700.
00000100:00000010:2.0:1788971961.295782:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6300.
00000100:00000010:2.0:1788971961.991484:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347ad80.
00000100:00000010:3.0:1788971961.994116:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4347a600.
00000100:00000010:3.0:1788971966.408198:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6d80.
02000000:00000010:3.0:1788971966.408227:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00.
00000100:00000010:3.0:1788971966.408265:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4a80.
02000000:00000010:3.0:1788971966.408269:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00.
02000000:00000010:2.0:1788971966.408379:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54059800.
00000400:00000200:2.0:1788971966.408397:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971966.408403:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160.
00000400:00000010:2.0:1788971966.408412:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971966.408421:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072256, portal 25
00000400:00000010:2.0:1788971966.408428:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8.
00000100:00000200:2.0:1788971966.408452:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179072256, offset 0
00000400:00000200:2.0:1788971966.408463:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971966.408474:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971966.408484:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971966.408491:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971966.408496:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971966.408500:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971966.408503:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971966.408507:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [351:351:256:4294967295] 192.168.203.124@tcp LPNI seq info [351:351:8:4294967295]
00000400:00000200:2.0:1788971966.408511:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971966.408522:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971966.408529:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971966.408535:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d300.
00000800:00000200:2.0:1788971966.408544:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971966.408553:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971966.408559:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971966.408590:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971966.408597:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7c00.
00000100:00000010:2.0:1788971966.408611:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f79c0.
02000000:00000010:2.0:1788971966.408642:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058800.
00000400:00000200:2.0:1788971966.408646:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971966.408649:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3420.
00000400:00000010:2.0:1788971966.408654:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18.
00000100:00000200:2.0:1788971966.408657:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072384, portal 10
00000400:00000010:2.0:1788971966.408663:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8.
00000100:00000200:2.0:1788971966.408665:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179072384, offset 0
00000400:00000200:2.0:1788971966.408669:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971966.408673:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971966.408677:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971966.408680:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971966.408682:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971966.408692:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971966.408694:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971966.408697:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [352:352:255:4294967295] 192.168.203.124@tcp LPNI seq info [352:352:7:4294967295]
00000400:00000200:2.0:1788971966.408702:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971966.408708:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971966.408712:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971966.408716:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d400.
00000800:00000200:2.0:1788971966.408720:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971966.408724:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971966.408726:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971966.408733:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971966.408986:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971966.409003:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d300.
00000400:00000200:3.0:1788971966.409012:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971966.409018:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971966.409024:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8
00000400:00000010:3.0:1788971966.409027:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8.
00000100:00000200:3.0:1788971966.409038:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c6d80 x1875872179072256/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971966.409593:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971966.409607:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d400.
00000400:00000200:3.0:1788971966.409612:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971966.409617:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971966.409622:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8
00000400:00000010:3.0:1788971966.409624:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8.
00000100:00000200:3.0:1788971966.409630:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c4a80 x1875872179072384/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971982 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971966.413154:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.413167:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971966.413177:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.413183:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.413190:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.413195:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ad80
00000400:00000200:3.0:1788971966.413208:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.413217:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1315 [1] + 224
00000400:00000200:3.0:1788971966.413234:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971966.413239:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971966.413258:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c4a80 x1875872179072384/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971982 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971966.416389:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3420.
00000400:00000200:2.0:1788971966.416406:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18
00000400:00000010:2.0:1788971966.416409:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18.
00000100:00000200:2.0:1788971966.416427:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c4a80 x1875872179072384/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971982 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971966.416441:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c4a80 x1875872179072384/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971982 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971966.416707:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.416715:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971966.416726:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.416731:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.416737:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.416741:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ad00
00000400:00000200:3.0:1788971966.416746:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971966.416753:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1305 [1] + 224
00000400:00000200:3.0:1788971966.416773:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971966.416779:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971966.416792:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c6d80 x1875872179072256/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971982 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971966.418345:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160.
00000400:00000200:2.0:1788971966.418360:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0
00000400:00000010:2.0:1788971966.418363:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971966.418376:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c6d80 x1875872179072256/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971982 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971966.418388:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c6d80 x1875872179072256/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971982 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971966.418420:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058800.
02000000:00000010:2.0:1788971966.418428:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00.
00000100:00000010:2.0:1788971966.418435:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4a80.
02000000:00000010:2.0:1788971966.418439:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54059800.
02000000:00000010:2.0:1788971966.418462:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00.
00000100:00000010:2.0:1788971966.418465:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6d80.
00000100:00000010:2.0:1788971967.111638:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7c00.
00000100:00000010:3.0:1788971967.113996:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f79c0.
00000100:00000010:0.0:1788971971.527571:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700.
02000000:00000010:0.0:1788971971.528022:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600.
00000100:00000010:0.0:1788971971.528074:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971971.528080:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400.
02000000:00000010:1.0:1788971971.528218:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3600.
00000400:00000200:1.0:1788971971.528228:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971971.528232:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971971.528239:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971971.528247:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072512, portal 25
00000400:00000010:1.0:1788971971.528253:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588bb0.
00000100:00000200:1.0:1788971971.528257:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179072512, offset 0
00000400:00000200:1.0:1788971971.528265:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971971.528274:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971971.528293:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971971.528298:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971971.528341:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971971.528345:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971971.528348:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971971.528352:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [353:353:256:4294967295] 192.168.203.124@tcp LPNI seq info [353:353:8:4294967295]
00000400:00000200:1.0:1788971971.528356:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971971.528382:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971971.528389:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971971.528394:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637a00.
00000800:00000200:1.0:1788971971.528419:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971971.528426:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971971.528430:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971971.528450:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971971.528454:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51127000.
00000100:00000010:1.0:1788971971.528466:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d93c0.
02000000:00000010:1.0:1788971971.528488:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400.
00000400:00000200:1.0:1788971971.528492:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971971.528494:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78.
00000400:00000010:1.0:1788971971.528498:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588440.
00000100:00000200:1.0:1788971971.528501:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072640, portal 10
00000400:00000010:1.0:1788971971.528505:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588908.
00000100:00000200:1.0:1788971971.528508:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179072640, offset 0
00000400:00000200:1.0:1788971971.528513:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971971.528516:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971971.528520:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971971.528523:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971971.528525:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971971.528527:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971971.528530:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971971.528533:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [354:354:255:4294967295] 192.168.203.124@tcp LPNI seq info [354:354:7:4294967295]
00000400:00000200:1.0:1788971971.528536:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971971.528542:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971971.528546:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971971.528549:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637100.
00000800:00000200:1.0:1788971971.528553:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971971.528557:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971971.528560:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971971.528566:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971971.528867:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971971.528872:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637a00.
00000400:00000200:3.0:1788971971.528881:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971971.528885:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971971.528891:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588bb0
00000400:00000010:3.0:1788971971.528892:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588bb0.
00000100:00000200:3.0:1788971971.528901:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c700 x1875872179072512/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971971.528973:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971971.528975:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637100.
00000400:00000200:3.0:1788971971.528979:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971971.528982:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971971.528985:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588908
00000400:00000010:3.0:1788971971.528987:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588908.
00000100:00000200:3.0:1788971971.528991:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179072640/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971987 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971971.531665:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.531674:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971971.531681:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.531686:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.531691:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.531694:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ae80
00000400:00000200:3.0:1788971971.531699:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.531705:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x132d [1] + 224
00000400:00000200:3.0:1788971971.531716:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971971.531721:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971971.531733:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179072640/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971987 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971971.531851:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78.
00000400:00000200:1.0:1788971971.531857:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588440
00000400:00000010:1.0:1788971971.531864:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588440.
00000100:00000200:1.0:1788971971.531871:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179072640/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971987 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971971.531879:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179072640/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971987 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971971.531908:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400.
02000000:00000010:1.0:1788971971.531914:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400.
00000100:00000010:1.0:1788971971.531918:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000400:00000200:3.0:1788971971.532644:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.532650:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971971.532655:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.532659:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.532662:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.532665:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ae00
00000400:00000200:3.0:1788971971.532669:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971971.532673:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1325 [1] + 224
00000400:00000200:3.0:1788971971.532683:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971971.532688:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971971.532694:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c700 x1875872179072512/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971987 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971971.532770:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971971.532775:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:1.0:1788971971.532777:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971971.532782:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c700 x1875872179072512/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971987 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971971.532789:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c700 x1875872179072512/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971987 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971971.532804:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3600.
02000000:00000010:1.0:1788971971.532818:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600.
00000100:00000010:1.0:1788971971.532822:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700.
00000100:00000010:2.0:1788971972.231424:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51127000.
00000100:00000010:3.0:1788971972.231654:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d93c0.
00000100:00000010:0.0:1788971976.647449:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700.
02000000:00000010:0.0:1788971976.647472:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00.
00000100:00000010:0.0:1788971976.647512:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100.
02000000:00000010:0.0:1788971976.647518:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800.
02000000:00000010:1.0:1788971976.647745:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3600.
00000400:00000200:1.0:1788971976.647755:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971976.647759:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971976.647765:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971976.647781:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072896, portal 10
00000400:00000010:1.0:1788971976.647788:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588440.
00000100:00000200:1.0:1788971976.647791:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179072896, offset 0
00000400:00000200:1.0:1788971976.647799:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971976.647808:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971976.647824:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971976.647830:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971976.647834:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971976.647837:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971976.647853:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971976.647857:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [355:355:256:4294967295] 192.168.203.124@tcp LPNI seq info [355:355:8:4294967295]
00000400:00000200:1.0:1788971976.647862:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971976.647872:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971976.647877:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971976.647882:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637100.
00000800:00000200:1.0:1788971976.647889:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971976.647896:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971976.647900:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971976.647922:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971976.647927:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9960.
00000100:00000010:1.0:1788971976.647947:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9600.
02000000:00000010:1.0:1788971976.647968:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400.
00000400:00000200:1.0:1788971976.647972:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971976.647975:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78.
00000400:00000010:1.0:1788971976.647979:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588220.
00000100:00000200:1.0:1788971976.647981:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179072768, portal 25
00000400:00000010:1.0:1788971976.647985:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588cc0.
00000100:00000200:1.0:1788971976.647987:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179072768, offset 0
00000400:00000200:1.0:1788971976.647991:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971976.647995:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971976.647999:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971976.648012:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971976.648014:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971976.648017:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971976.648019:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971976.648022:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [356:356:255:4294967295] 192.168.203.124@tcp LPNI seq info [356:356:7:4294967295]
00000400:00000200:1.0:1788971976.648025:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971976.648031:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971976.648035:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971976.648045:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51637a00.
00000800:00000200:1.0:1788971976.648049:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971976.648053:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971976.648056:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51637a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971976.648061:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971976.648453:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971976.648459:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637100.
00000400:00000200:3.0:1788971976.648467:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971976.648472:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971976.648477:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588440
00000400:00000010:3.0:1788971976.648479:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588440.
00000100:00000200:3.0:1788971976.648487:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353f100 x1875872179072896/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971976.648555:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971976.648558:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51637a00.
00000400:00000200:3.0:1788971976.648561:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971976.648564:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971976.648566:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588cc0
00000400:00000010:3.0:1788971976.648568:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588cc0.
00000100:00000200:3.0:1788971976.648573:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353c700 x1875872179072768/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971992 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971976.651247:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651267:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971976.651276:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651280:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651285:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651289:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0af00
00000400:00000200:3.0:1788971976.651295:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651331:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x133d [1] + 224
00000400:00000200:3.0:1788971976.651813:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971976.651818:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971976.651832:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353c700 x1875872179072768/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971992 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971976.651871:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651875:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971976.651879:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651882:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651885:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651888:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0af80
00000400:00000200:3.0:1788971976.651891:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971976.651896:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1335 [1] + 224
00000400:00000200:3.0:1788971976.651903:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971976.651906:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971976.651913:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353f100 x1875872179072896/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971992 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971976.654091:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78.
00000400:00000200:1.0:1788971976.654104:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588220
00000400:00000010:1.0:1788971976.654106:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588220.
00000100:00000200:1.0:1788971976.654120:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353c700 x1875872179072768/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971992 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971976.654141:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353c700 x1875872179072768/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971992 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971976.654165:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971976.654169:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:1.0:1788971976.654170:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971976.654175:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353f100 x1875872179072896/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971992 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971976.654182:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353f100 x1875872179072896/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971992 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971976.654201:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400.
02000000:00000010:1.0:1788971976.654217:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00.
00000100:00000010:1.0:1788971976.654223:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700.
02000000:00000010:1.0:1788971976.654227:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3600.
02000000:00000010:1.0:1788971976.654230:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800.
00000100:00000010:1.0:1788971976.654232:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100.
00000100:00000010:2.0:1788971977.351392:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9960.
00000100:00000010:3.0:1788971977.351521:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9600.
00000001:02000400:1.0:1788971979.948039:0:111395:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_300: @@@@@@ FAIL: test_300 failed with 2
Debug log: 3202 lines, 3202 kept, 0 dropped, 0 bad.