00000100:00000010:3.0F:1788972570.567648:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380.
02000000:00000010:3.0:1788972570.567700:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1100.
00000100:00000010:3.0:1788972570.567752:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800.
02000000:00000010:3.0:1788972570.567759:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1900.
02000000:00000010:3.0:1788972570.567806:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0c00.
00000400:00000200:3.0:1788972570.567818:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788972570.567824:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e784d0.
00000400:00000010:3.0:1788972570.567833:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788972570.567841:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179102720, portal 10
00000400:00000010:3.0:1788972570.567851:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788972570.567854:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179102720, offset 0
00000400:00000200:3.0:1788972570.567866:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788972570.567879:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.567889:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788972570.567896:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972570.567901:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788972570.567906:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972570.567909:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788972570.567913:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [588:588:256:4294967295] 192.168.203.124@tcp LPNI seq info [588:588:8:4294967295]
00000400:00000200:3.0:1788972570.567918:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.567931:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972570.567938:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788972570.567944:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1500.
00000800:00000200:3.0:1788972570.567953:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788972570.567963:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788972570.567968:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972570.567988:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788972570.567994:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f919c0.
00000100:00000010:3.0:1788972570.568008:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f911e0.
02000000:00000010:3.0:1788972570.568032:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f1e00.
00000400:00000200:3.0:1788972570.568037:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788972570.568040:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78160.
00000400:00000010:3.0:1788972570.568051:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788972570.568054:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179102592, portal 25
00000400:00000010:3.0:1788972570.568058:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110.
00000100:00000200:3.0:1788972570.568060:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179102592, offset 0
00000400:00000200:3.0:1788972570.568065:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788972570.568070:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.568073:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788972570.568076:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972570.568078:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788972570.568081:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972570.568084:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788972570.568087:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [589:589:255:4294967295] 192.168.203.124@tcp LPNI seq info [589:589:7:4294967295]
00000400:00000200:3.0:1788972570.568091:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.568098:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972570.568102:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788972570.568106:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1b00.
00000800:00000200:3.0:1788972570.568110:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788972570.568114:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788972570.568117:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972570.568122:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788972570.568197:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972570.568202:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1500.
00000400:00000200:3.0:1788972570.568210:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972570.568216:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972570.568224:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480815d8
00000400:00000010:3.0:1788972570.568227:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788972570.568238:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c7800 x1875872179102720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972570.568360:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972570.568363:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1b00.
00000400:00000200:3.0:1788972570.568367:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972570.568376:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972570.568380:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110
00000400:00000010:3.0:1788972570.568382:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110.
00000100:00000200:3.0:1788972570.568387:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c4380 x1875872179102592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972570.573185:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.573198:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972570.573208:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.573214:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.573220:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.573225:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e12380
00000400:00000200:3.0:1788972570.573231:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.573237:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1e2d [1] + 224
00000400:00000200:3.0:1788972570.573263:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972570.573268:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972570.573287:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c4380 x1875872179102592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972586 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972570.573386:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78160.
00000400:00000200:3.0:1788972570.573393:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081088
00000400:00000010:3.0:1788972570.573396:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788972570.573403:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c4380 x1875872179102592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972586 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972570.573413:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c4380 x1875872179102592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972586 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972570.573453:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f1e00.
02000000:00000010:3.0:1788972570.573460:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1100.
00000100:00000010:3.0:1788972570.573467:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380.
00000400:00000200:3.0:1788972570.577056:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.577070:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972570.577081:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.577096:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.577104:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.577109:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e12400
00000400:00000200:3.0:1788972570.577116:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972570.577123:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1e1d [1] + 224
00000400:00000200:3.0:1788972570.577139:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972570.577145:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972570.577163:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c7800 x1875872179102720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972586 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972570.577212:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e784d0.
00000400:00000200:3.0:1788972570.577217:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081220
00000400:00000010:3.0:1788972570.577220:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788972570.577227:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c7800 x1875872179102720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972586 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972570.577237:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c7800 x1875872179102720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972586 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972570.577275:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0c00.
02000000:00000010:3.0:1788972570.577280:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1900.
00000100:00000010:3.0:1788972570.577286:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800.
00000100:00000010:3.0:1788972571.271503:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f911e0.
00000100:00000010:2.0F:1788972571.271684:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f919c0.
00000100:00000010:2.0:1788972575.695860:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411180.
02000000:00000010:2.0:1788972575.695896:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4605b600.
00000100:00000010:2.0:1788972575.695959:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412680.
02000000:00000010:2.0:1788972575.695967:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4605b500.
02000000:00000010:2.0:1788972575.696015:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549200.
00000400:00000200:2.0:1788972575.696028:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788972575.696034:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3d68.
00000400:00000010:2.0:1788972575.696043:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0550.
00000100:00000200:2.0:1788972575.696052:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179102976, portal 10
00000400:00000010:2.0:1788972575.696062:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0000.
00000100:00000200:2.0:1788972575.696065:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179102976, offset 0
00000400:00000200:2.0:1788972575.696077:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788972575.696089:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.696099:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788972575.696105:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972575.696110:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788972575.696115:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972575.696118:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788972575.696121:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [590:590:256:4294967295] 192.168.203.124@tcp LPNI seq info [590:590:8:4294967295]
00000400:00000200:2.0:1788972575.696126:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.696138:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972575.696145:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788972575.696151:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f700.
00000800:00000200:2.0:1788972575.696159:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788972575.696168:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788972575.696185:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f700 type 1, nob 320 niov 1 nkiov 1
02000000:00000010:3.0:1788972575.696199:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0a00.
00000400:00000200:2.0:1788972575.696206:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972575.696210:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788972575.696216:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e784d0.
00000100:00000010:2.0:1788972575.696217:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f112a0.
00000400:00000010:3.0:1788972575.696221:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788972575.696226:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179102848, portal 25
00000400:00000010:3.0:1788972575.696231:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788972575.696234:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179102848, offset 0
00000100:00000010:2.0:1788972575.696234:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f111e0.
00000400:00000200:3.0:1788972575.696245:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788972575.696249:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972575.696254:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788972575.696275:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972575.696278:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788972575.696280:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972575.696283:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788972575.696286:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [591:591:255:4294967295] 192.168.203.124@tcp LPNI seq info [591:591:7:4294967295]
00000400:00000200:3.0:1788972575.696290:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972575.696296:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972575.696341:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788972575.696346:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100.
00000800:00000200:3.0:1788972575.696351:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788972575.696357:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788972575.696360:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972575.696365:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788972575.696431:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972575.696437:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f700.
00000400:00000200:2.0:1788972575.696448:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972575.696455:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788972575.696461:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0000
00000400:00000010:2.0:1788972575.696465:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0000.
00000100:00000200:2.0:1788972575.696477:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412680 x1875872179102976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788972575.696508:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972575.696511:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100.
00000400:00000200:2.0:1788972575.696515:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972575.696530:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788972575.696533:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081088
00000400:00000010:2.0:1788972575.696535:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081088.
00000100:00000200:2.0:1788972575.696540:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43411180 x1875872179102848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788972575.702246:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702259:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788972575.702270:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702277:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702284:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702288:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e12480
00000400:00000200:2.0:1788972575.702295:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702361:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1e4d [1] + 224
00000400:00000200:2.0:1788972575.702378:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972575.702384:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788972575.702402:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43411180 x1875872179102848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972591 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788972575.702441:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702444:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788972575.702450:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702452:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702473:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702476:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e12500
00000400:00000200:2.0:1788972575.702480:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972575.702485:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1e3d [1] + 224
00000400:00000200:2.0:1788972575.702497:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972575.702500:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788972575.702507:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412680 x1875872179102976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972591 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972575.702566:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3d68.
00000400:00000200:2.0:1788972575.702571:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0550
00000400:00000010:2.0:1788972575.702574:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0550.
00000100:00000200:2.0:1788972575.702580:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412680 x1875872179102976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972575.702589:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412680 x1875872179102976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972591 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972575.702630:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549200.
02000000:00000010:2.0:1788972575.702638:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4605b500.
00000100:00000010:2.0:1788972575.702646:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412680.
00000400:00000010:3.0:1788972575.704910:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e784d0.
00000400:00000200:3.0:1788972575.704925:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081220
00000400:00000010:3.0:1788972575.704928:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788972575.704944:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43411180 x1875872179102848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972575.704956:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43411180 x1875872179102848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972591 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972575.704987:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0a00.
02000000:00000010:3.0:1788972575.704994:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4605b600.
00000100:00000010:3.0:1788972575.705000:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411180.
00000100:00000010:3.0:1788972576.391385:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f111e0.
00000100:00000010:2.0:1788972576.392206:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f112a0.
00000001:02000400:0.0F:1788972580.099590:0:123199:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 475: per-operation latency stats populate and reset ========================================================== 12:49:37 (1788972577)
00000001:02000400:1.0F:1788972580.280094:0:123204:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000100:00000010:1.0:1788972580.807697:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4a80.
02000000:00000010:1.0:1788972580.807732:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49714e00.
02000000:00000010:1.0:1788972580.815427:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058e00.
00000400:00000200:1.0:1788972580.815450:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788972580.815456:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10.
00000400:00000010:1.0:1788972580.815465:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880.
00000100:00000200:1.0:1788972580.815476:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179103104, portal 25
00000400:00000010:1.0:1788972580.815492:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445885d8.
00000100:00000200:1.0:1788972580.815498:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179103104, offset 0
00000400:00000200:1.0:1788972580.815509:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788972580.815521:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788972580.815531:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788972580.815537:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972580.815542:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788972580.815547:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972580.815550:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788972580.815554:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [592:592:256:4294967295] 192.168.203.124@tcp LPNI seq info [592:592:8:4294967295]
00000400:00000200:1.0:1788972580.815558:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788972580.815570:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972580.815578:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788972580.815584:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49714400.
00000800:00000200:1.0:1788972580.815592:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788972580.815601:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788972580.815606:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49714400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972580.816375:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788972580.816385:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d97e0.
00000100:00000010:1.0:1788972580.816401:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d95a0.
00000800:00000200:3.0:1788972580.817281:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972580.817290:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49714400.
00000400:00000200:3.0:1788972580.817321:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972580.817331:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972580.817340:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445885d8
00000400:00000010:3.0:1788972580.817343:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445885d8.
00000100:00000200:3.0:1788972580.817368:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb4a80 x1875872179103104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:2.0:1788972580.824984:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412680.
02000000:00000010:2.0:1788972580.825019:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb514fc600.
00000400:00000200:3.0:1788972580.825076:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.825084:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
02000000:00000010:2.0:1788972580.825087:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50f40000.
00000400:00000200:3.0:1788972580.825093:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972580.825094:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788972580.825098:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3d68.
00000400:00000200:3.0:1788972580.825099:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.825105:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788972580.825106:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0550.
00000400:00000200:3.0:1788972580.825109:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e12580
00000400:00000200:3.0:1788972580.825114:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000200:2.0:1788972580.825115:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179103232, portal 10
00000400:00000200:3.0:1788972580.825119:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1e5d [1] + 224
00000400:00000010:2.0:1788972580.825123:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0000.
00000100:00000200:2.0:1788972580.825126:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179103232, offset 0
00000400:00000200:2.0:1788972580.825132:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788972580.825139:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.825147:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972580.825152:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788972580.825154:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788972580.825160:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972580.825169:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788972580.825176:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972580.825179:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788972580.825183:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [593:593:256:4294967295] 192.168.203.124@tcp LPNI seq info [593:593:8:4294967295]
00000400:00000200:2.0:1788972580.825188:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972580.825197:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972580.825203:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788972580.825208:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb514fc400.
00000800:00000200:2.0:1788972580.826078:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788972580.826088:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788972580.826094:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb514fc400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972580.826123:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788972580.833643:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972580.833658:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb514fc400.
00000400:00000200:2.0:1788972580.833666:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972580.833677:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972580.834639:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb4a80 x1875872179103104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972596 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972580.834775:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10.
00000400:00000200:1.0:1788972580.834781:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880
00000400:00000010:1.0:1788972580.834784:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880.
00000100:00000200:1.0:1788972580.834791:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb4a80 x1875872179103104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972596 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972580.834811:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb4a80 x1875872179103104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972596 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788972580.834845:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058e00.
02000000:00000010:1.0:1788972580.834852:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49714e00.
00000100:00000010:1.0:1788972580.834860:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4a80.
00000400:00000200:3.0:1788972580.837536:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.837547:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972580.837558:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.837563:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.837570:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.837575:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e12600
00000400:00000200:3.0:1788972580.837582:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972580.837590:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1e65 [1] + 224
00000400:00000200:3.0:1788972580.837605:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972580.837609:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972580.837625:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412680 x1875872179103232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972580.837677:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3d68.
00000400:00000200:3.0:1788972580.837682:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0550
00000400:00000010:3.0:1788972580.837685:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0550.
00000100:00000200:3.0:1788972580.837699:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412680 x1875872179103232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972596 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972580.837708:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412680 x1875872179103232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972596 ref 2 fl Rpc:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788972580.841574:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0000
00000400:00000010:2.0:1788972580.841586:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0000.
00000100:00000200:2.0:1788972580.841608:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412680 x1875872179103232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972596 ref 2 fl UnregRPC:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972580.843946:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50f40000.
02000000:00000010:3.0:1788972580.843961:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb514fc600.
00000100:00000010:3.0:1788972580.843967:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412680.
00000100:00000010:2.0:1788972581.513805:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d97e0.
00000100:00000010:3.0:1788972581.521877:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d95a0.
00000001:02000400:2.0:1788972582.533153:0:123340:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_475: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.