00000100:00000010:0.0F:1788971817.928019:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80.
02000000:00000010:0.0:1788971817.928045:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500.
00000100:00000010:0.0:1788971817.928084:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00.
02000000:00000010:0.0:1788971817.928089:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55d00.
02000000:00000010:0.0:1788971817.928119:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017200.
00000400:00000200:0.0:1788971817.928128:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971817.928132:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580.
00000400:00000010:0.0:1788971817.928138:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000.
00000100:00000200:0.0:1788971817.928145:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064960, portal 10
00000400:00000010:0.0:1788971817.928151:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084088.
00000100:00000200:0.0:1788971817.928154:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179064960, offset 0
00000400:00000200:0.0:1788971817.928161:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971817.928169:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971817.928175:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971817.928180:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971817.928184:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971817.928187:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971817.928190:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971817.928193:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [293:293:256:4294967295] 192.168.203.124@tcp LPNI seq info [293:293:8:4294967295]
00000400:00000200:0.0:1788971817.928197:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971817.928207:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971817.928213:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971817.928217:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800.
00000800:00000200:0.0:1788971817.928223:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971817.928230:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971817.928234:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971817.928255:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971817.928260:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84660.
00000100:00000010:0.0:1788971817.928269:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84c60.
02000000:00000010:0.0:1788971817.928287:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016c00.
00000400:00000200:0.0:1788971817.928291:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971817.928294:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163d68.
00000400:00000010:0.0:1788971817.928320:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084bb0.
00000100:00000200:0.0:1788971817.928323:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064832, portal 25
00000400:00000010:0.0:1788971817.928326:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220.
00000100:00000200:0.0:1788971817.928328:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179064832, offset 0
00000400:00000200:0.0:1788971817.928333:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971817.928336:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971817.928339:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971817.928342:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971817.928344:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971817.928346:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971817.928348:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971817.928363:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [294:294:255:4294967295] 192.168.203.124@tcp LPNI seq info [294:294:7:4294967295]
00000400:00000200:0.0:1788971817.928367:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971817.928372:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971817.928376:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971817.928379:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55f00.
00000800:00000200:0.0:1788971817.928382:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0F:1788971817.928385:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000200:0.0:1788971817.928386:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971817.928389:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971817.928393:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000010:2.0:1788971817.928394:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800.
00000400:00000200:2.0:1788971817.928408:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971817.928413:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971817.928418:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084088
00000400:00000010:2.0:1788971817.928421:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084088.
00000100:00000200:2.0:1788971817.928430:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353ea00 x1875872179064960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971817.928454:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971817.928457:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55f00.
00000400:00000200:2.0:1788971817.928460:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971817.928463:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971817.928465:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220
00000400:00000010:2.0:1788971817.928467:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220.
00000100:00000200:2.0:1788971817.928471:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353fb80 x1875872179064832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971833 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0F:1788971817.933045:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971817.933061:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971817.933070:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971817.933074:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971817.933079:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971817.933082:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09000
00000400:00000200:3.0:1788971817.933087:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971817.933093:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x101d [1] + 224
00000400:00000200:3.0:1788971817.933104:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971817.933109:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971817.933123:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353fb80 x1875872179064832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971833 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0F:1788971817.933680:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163d68.
00000400:00000200:1.0:1788971817.933751:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084bb0
00000400:00000010:1.0:1788971817.933755:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084bb0.
00000100:00000200:1.0:1788971817.933763:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353fb80 x1875872179064832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971833 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971817.933773:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353fb80 x1875872179064832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971833 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971817.933859:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016c00.
02000000:00000010:1.0:1788971817.933865:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500.
00000100:00000010:1.0:1788971817.933870:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80.
00000400:00000200:2.0:1788971817.934482:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971817.934488:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971817.934495:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971817.934500:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971817.934503:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971817.934507:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09080
00000400:00000200:2.0:1788971817.934512:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971817.934516:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1015 [1] + 224
00000400:00000200:2.0:1788971817.934530:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971817.934534:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971817.934542:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353ea00 x1875872179064960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971833 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971817.934879:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580.
00000400:00000200:1.0:1788971817.934884:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000
00000400:00000010:1.0:1788971817.934886:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000.
00000100:00000200:1.0:1788971817.934893:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353ea00 x1875872179064960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971833 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971817.934911:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353ea00 x1875872179064960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971833 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971817.934932:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017200.
02000000:00000010:1.0:1788971817.934937:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55d00.
00000100:00000010:1.0:1788971817.934941:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00.
00000100:00000010:3.0:1788971818.632408:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84c60.
00000100:00000010:2.0:1788971818.633109:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84660.
00000100:00000010:3.0:1788971823.047551:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00.
02000000:00000010:3.0:1788971823.047581:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200.
00000100:00000010:3.0:1788971823.047621:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00.
02000000:00000010:3.0:1788971823.047625:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700.
02000000:00000010:2.0:1788971823.047735:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5a00.
00000400:00000200:2.0:1788971823.047753:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971823.047787:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3630.
00000400:00000010:2.0:1788971823.047796:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0bb0.
00000100:00000200:2.0:1788971823.047805:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065088, portal 25
00000400:00000010:2.0:1788971823.047811:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0000.
00000100:00000200:2.0:1788971823.047814:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179065088, offset 0
00000400:00000200:2.0:1788971823.047825:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971823.047836:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.047845:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971823.047851:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971823.047856:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971823.047860:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971823.047878:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971823.047883:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [295:295:256:4294967295] 192.168.203.124@tcp LPNI seq info [295:295:8:4294967295]
00000400:00000200:2.0:1788971823.047888:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.047899:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971823.047907:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971823.047913:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00.
00000800:00000200:2.0:1788971823.047921:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971823.047930:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971823.047935:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971823.047958:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971823.047964:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f74e0.
00000100:00000010:2.0:1788971823.047977:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7f60.
02000000:00000010:2.0:1788971823.048000:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5600.
00000400:00000200:2.0:1788971823.048005:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971823.048007:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3dc0.
00000400:00000010:2.0:1788971823.048011:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660.
00000100:00000200:2.0:1788971823.048014:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065216, portal 10
00000400:00000010:2.0:1788971823.048019:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0550.
00000100:00000200:2.0:1788971823.048021:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179065216, offset 0
00000400:00000200:2.0:1788971823.048026:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971823.048036:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.048039:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971823.048042:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971823.048044:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971823.048046:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971823.048049:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971823.048051:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [296:296:255:4294967295] 192.168.203.124@tcp LPNI seq info [296:296:7:4294967295]
00000400:00000200:2.0:1788971823.048055:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.048060:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971823.048064:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971823.048069:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ed00.
00000800:00000200:2.0:1788971823.048072:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971823.048076:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971823.048079:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ed00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971823.048084:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788971823.048189:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971823.048195:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00.
00000400:00000200:3.0:1788971823.048204:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971823.048210:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971823.048216:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0000
00000400:00000010:3.0:1788971823.048219:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0000.
00000100:00000200:3.0:1788971823.048230:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c4e00 x1875872179065088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971823.048356:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971823.048359:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ed00.
00000400:00000200:3.0:1788971823.048363:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971823.048367:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971823.048369:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0550
00000400:00000010:3.0:1788971823.048371:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0550.
00000100:00000200:3.0:1788971823.048376:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c6a00 x1875872179065216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971839 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971823.057001:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057013:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971823.057024:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057031:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057037:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057042:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09180
00000400:00000200:2.0:1788971823.057055:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057062:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1035 [1] + 224
00000400:00000200:2.0:1788971823.057077:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971823.057083:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971823.057101:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c6a00 x1875872179065216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971839 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971823.057155:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057159:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971823.057164:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057187:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057190:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057193:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09100
00000400:00000200:2.0:1788971823.057197:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971823.057214:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1025 [1] + 224
00000400:00000200:2.0:1788971823.057226:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971823.057230:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971823.057238:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c4e00 x1875872179065088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971839 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971823.057557:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3dc0.
00000400:00000200:3.0:1788971823.057564:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660
00000400:00000010:3.0:1788971823.057567:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660.
00000100:00000200:3.0:1788971823.057574:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c6a00 x1875872179065216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971839 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971823.057584:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c6a00 x1875872179065216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971839 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971823.057618:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3630.
00000400:00000200:3.0:1788971823.057620:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0bb0
00000400:00000010:3.0:1788971823.057631:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0bb0.
00000100:00000200:3.0:1788971823.057636:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c4e00 x1875872179065088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971839 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971823.057643:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c4e00 x1875872179065088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971839 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971823.058050:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5600.
02000000:00000010:3.0:1788971823.058059:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700.
00000100:00000010:3.0:1788971823.058066:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00.
02000000:00000010:3.0:1788971823.058070:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5a00.
02000000:00000010:3.0:1788971823.058073:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200.
00000100:00000010:3.0:1788971823.058075:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00.
00000100:00000010:3.0:1788971823.753343:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7f60.
00000100:00000010:2.0:1788971823.753822:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f74e0.
00000100:00000010:0.0:1788971828.168164:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00.
02000000:00000010:0.0:1788971828.168183:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300.
00000100:00000010:0.0:1788971828.168211:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80.
02000000:00000010:0.0:1788971828.168215:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400.
02000000:00000010:0.0:1788971828.168245:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aac00.
00000400:00000200:0.0:1788971828.168254:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971828.168258:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481639f8.
00000400:00000010:0.0:1788971828.168264:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084908.
00000100:00000200:0.0:1788971828.168271:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065344, portal 25
00000400:00000010:0.0:1788971828.168276:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480843b8.
00000100:00000200:0.0:1788971828.168279:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179065344, offset 0
00000400:00000200:0.0:1788971828.168295:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971828.168320:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971828.168328:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971828.168333:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971828.168337:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971828.168341:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971828.168344:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971828.168347:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [297:297:256:4294967295] 192.168.203.124@tcp LPNI seq info [297:297:8:4294967295]
00000400:00000200:0.0:1788971828.168352:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971828.168360:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971828.168366:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971828.168372:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55c00.
00000800:00000200:0.0:1788971828.168444:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971828.168453:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971828.168458:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55c00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971828.168517:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788971828.168523:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e847e0.
00000100:00000010:0.0:1788971828.168533:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84360.
02000000:00000010:0.0:1788971828.168553:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000.
00000400:00000200:0.0:1788971828.168557:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788971828.168560:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163b58.
00000400:00000010:0.0:1788971828.168564:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084440.
00000100:00000200:0.0:1788971828.168567:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179065472, portal 10
00000400:00000010:0.0:1788971828.168572:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550.
00000100:00000200:0.0:1788971828.168575:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179065472, offset 0
00000400:00000200:0.0:1788971828.168580:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788971828.168583:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971828.168587:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788971828.168589:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971828.168591:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788971828.168593:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971828.168602:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788971828.168605:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [298:298:255:4294967295] 192.168.203.124@tcp LPNI seq info [298:298:7:4294967295]
00000400:00000200:0.0:1788971828.168609:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788971828.168615:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971828.168619:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788971828.168622:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00.
00000800:00000200:0.0:1788971828.168626:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788971828.168630:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788971828.168633:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971828.168639:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971828.175249:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971828.175260:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55c00.
00000400:00000200:2.0:1788971828.175266:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971828.175275:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971828.175283:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480843b8
00000400:00000010:2.0:1788971828.175286:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480843b8.
00000100:00000200:2.0:1788971828.175297:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353ea00 x1875872179065344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971828.175394:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971828.175397:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00.
00000400:00000200:2.0:1788971828.175401:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971828.175404:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971828.175407:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550
00000400:00000010:2.0:1788971828.175409:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550.
00000100:00000200:2.0:1788971828.175414:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb4353fb80 x1875872179065472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971844 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971828.177602:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177611:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971828.177619:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177623:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177628:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177644:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e09200
00000400:00000200:2.0:1788971828.177648:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177654:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1045 [1] + 224
00000400:00000200:2.0:1788971828.177665:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971828.177668:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971828.177679:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353ea00 x1875872179065344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971844 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971828.177719:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177723:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971828.177728:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177731:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177734:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177736:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e09280
00000400:00000200:2.0:1788971828.177740:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971828.177744:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x104d [1] + 224
00000400:00000200:2.0:1788971828.177752:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971828.177755:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971828.177760:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb4353fb80 x1875872179065472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971844 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971828.177849:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163b58.
00000400:00000200:1.0:1788971828.177854:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084440
00000400:00000010:1.0:1788971828.177857:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084440.
00000100:00000200:1.0:1788971828.177863:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353fb80 x1875872179065472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971844 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971828.177872:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353fb80 x1875872179065472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971844 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971828.177896:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481639f8.
00000400:00000200:1.0:1788971828.177899:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084908
00000400:00000010:1.0:1788971828.177900:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084908.
00000100:00000200:1.0:1788971828.177904:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb4353ea00 x1875872179065344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971844 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971828.177917:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb4353ea00 x1875872179065344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971844 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971828.177941:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000.
02000000:00000010:1.0:1788971828.177947:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400.
00000100:00000010:1.0:1788971828.177951:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80.
02000000:00000010:1.0:1788971828.177964:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aac00.
02000000:00000010:1.0:1788971828.177967:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300.
00000100:00000010:1.0:1788971828.177969:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00.
00000001:02000400:0.0:1788971828.333394:0:110134:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 291: load lnet using custom accept port for bulk tcp connections ========================================================== 12:37:06 (1788971826)
00000001:02000400:2.0:1788971828.759019:0:110149:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000100:00000010:2.0:1788971828.871435:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e847e0.
00000100:00000010:3.0:1788971828.871737:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84360.
00000001:02000400:1.0:1788971830.035245:0:110275:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_291: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.