00000100:00000010:3.0F:1788972509.128479:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5c00.
02000000:00000010:3.0:1788972509.128508:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51637100.
00000100:00000010:3.0:1788972509.128550:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4700.
02000000:00000010:3.0:1788972509.128556:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51637a00.
02000000:00000010:3.0:1788972509.128591:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058e00.
00000400:00000200:3.0:1788972509.128600:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788972509.128604:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789a0.
00000400:00000010:3.0:1788972509.128610:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081088.
00000100:00000200:3.0:1788972509.128617:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179099648, portal 10
00000400:00000010:3.0:1788972509.128625:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788972509.128628:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179099648, offset 0
00000400:00000200:3.0:1788972509.128635:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:3.0:1788972509.128643:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.128650:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788972509.128655:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972509.128659:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788972509.128664:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972509.128666:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788972509.128670:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [564:564:256:4294967295] 192.168.203.124@tcp LPNI seq info [564:564:8:4294967295]
00000400:00000200:3.0:1788972509.128674:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.128683:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972509.128689:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788972509.128694:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d000.
00000800:00000200:3.0:1788972509.128701:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788972509.128708:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788972509.128713:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972509.128727:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788972509.128732:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91d20.
02000000:00000010:2.0F:1788972509.128887:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a800.
00000400:00000200:2.0:1788972509.128899:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788972509.128902:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160.
00000400:00000010:2.0:1788972509.128906:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788972509.128910:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179099520, portal 25
00000400:00000010:2.0:1788972509.128914:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8.
00000100:00000200:2.0:1788972509.128916:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179099520, offset 0
00000400:00000200:2.0:1788972509.128921:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788972509.128925:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.128929:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788972509.128932:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972509.128935:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788972509.128937:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972509.128939:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788972509.128943:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [565:565:255:4294967295] 192.168.203.124@tcp LPNI seq info [565:565:7:4294967295]
00000400:00000200:2.0:1788972509.128947:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.128953:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972509.128957:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788972509.128961:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1f00.
00000800:00000200:2.0:1788972509.128964:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788972509.128969:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788972509.128971:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972509.128976:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:3.0:1788972509.129033:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f911e0.
00000800:00000200:3.0:1788972509.129087:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972509.129090:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d000.
00000400:00000200:3.0:1788972509.129095:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972509.129106:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972509.129112:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480815d8
00000400:00000010:3.0:1788972509.129114:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788972509.129123:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c4700 x1875872179099648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972509.129860:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972509.129865:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1f00.
00000400:00000200:3.0:1788972509.129869:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972509.129874:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972509.129879:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8
00000400:00000010:3.0:1788972509.129880:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8.
00000100:00000200:3.0:1788972509.129887:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c5c00 x1875872179099520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972509.131761:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.131771:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972509.131778:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.131782:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.131788:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.131799:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e11800
00000400:00000200:3.0:1788972509.131805:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972509.131811:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1ced [1] + 224
00000400:00000200:3.0:1788972509.131824:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972509.131830:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972509.131842:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c4700 x1875872179099648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972525 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972509.131970:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789a0.
00000400:00000200:2.0:1788972509.131976:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081088
00000400:00000010:2.0:1788972509.131978:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081088.
00000100:00000200:2.0:1788972509.131985:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c4700 x1875872179099648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972525 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972509.132003:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c4700 x1875872179099648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972525 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972509.132035:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058e00.
02000000:00000010:2.0:1788972509.132050:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51637a00.
00000100:00000010:2.0:1788972509.132056:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4700.
00000400:00000200:2.0:1788972509.140610:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.140620:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788972509.140628:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.140633:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.140638:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.140642:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e11780
00000400:00000200:2.0:1788972509.140648:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972509.140654:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1cfd [1] + 224
00000400:00000200:2.0:1788972509.140668:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972509.140672:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788972509.140687:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c5c00 x1875872179099520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972525 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972509.140743:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160.
00000400:00000200:2.0:1788972509.140749:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8
00000400:00000010:2.0:1788972509.140751:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788972509.140758:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c5c00 x1875872179099520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972525 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972509.140768:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c5c00 x1875872179099520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972525 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972509.140802:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a800.
02000000:00000010:2.0:1788972509.140808:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51637100.
00000100:00000010:2.0:1788972509.140814:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5c00.
00000100:00000010:3.0:1788972509.832102:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f911e0.
00000100:00000010:2.0:1788972509.833133:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91d20.
00000100:00000010:1.0F:1788972514.248186:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4000.
02000000:00000010:1.0:1788972514.248211:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49714300.
00000100:00000010:1.0:1788972514.248242:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb5880.
02000000:00000010:1.0:1788972514.248246:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49714100.
02000000:00000010:1.0:1788972514.248287:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d68e00.
00000400:00000200:1.0:1788972514.248296:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
02000000:00000010:0.0F:1788972514.248498:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548000.
00000400:00000200:0.0:1788972514.248508:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:0.0:1788972514.248512:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163840.
00000400:00000010:0.0:1788972514.248520:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084dd0.
00000100:00000200:0.0:1788972514.248526:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179099904, portal 10
00000400:00000010:0.0:1788972514.248533:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198.
00000100:00000200:0.0:1788972514.248537:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179099904, offset 0
00000400:00000200:0.0:1788972514.248545:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:0.0:1788972514.248553:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788972514.248560:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:0.0:1788972514.248565:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788972514.248569:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:0.0:1788972514.248572:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788972514.248575:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:0.0:1788972514.248579:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [566:566:256:4294967295] 192.168.203.124@tcp LPNI seq info [566:566:8:4294967295]
00000400:00000200:0.0:1788972514.248583:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:0.0:1788972514.248592:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788972514.248599:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:0.0:1788972514.248604:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55900.
00000800:00000200:0.0:1788972514.248610:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:0.0:1788972514.248618:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:0.0:1788972514.248622:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788972514.248643:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:0.0:1788972514.248648:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969d20.
00000100:00000010:0.0:1788972514.248658:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449698a0.
00000800:00000200:3.0:1788972514.250864:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972514.250872:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55900.
00000400:00000200:3.0:1788972514.250879:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972514.250887:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972514.250894:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198
00000400:00000010:3.0:1788972514.250906:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198.
00000100:00000200:3.0:1788972514.250916:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb5880 x1875872179099904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972514.252036:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10.
00000400:00000010:1.0:1788972514.252046:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880.
00000100:00000200:1.0:1788972514.252052:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179099776, portal 25
00000400:00000010:1.0:1788972514.252058:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588bb0.
00000100:00000200:1.0:1788972514.252060:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179099776, offset 0
00000400:00000200:1.0:1788972514.252069:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788972514.252077:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788972514.252084:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788972514.252089:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972514.252093:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788972514.252096:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972514.252099:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788972514.252102:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [567:567:256:4294967295] 192.168.203.124@tcp LPNI seq info [567:567:8:4294967295]
00000400:00000200:1.0:1788972514.252106:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788972514.252115:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972514.252121:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788972514.252126:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49714400.
00000800:00000200:1.0:1788972514.252133:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788972514.252139:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788972514.252142:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49714400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972514.252161:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0:1788972514.252530:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972514.252534:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49714400.
00000400:00000200:3.0:1788972514.252539:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972514.252544:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788972514.252548:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588bb0
00000400:00000010:3.0:1788972514.252550:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588bb0.
00000100:00000200:3.0:1788972514.252558:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb4000 x1875872179099776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972514.253543:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.253550:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972514.253558:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.253563:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.253567:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.253570:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e11900
00000400:00000200:3.0:1788972514.253575:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.253580:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1d0d [1] + 224
00000400:00000200:3.0:1788972514.253592:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972514.253596:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972514.253605:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb5880 x1875872179099904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972530 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788972514.253859:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163840.
00000400:00000200:0.0:1788972514.253865:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084dd0
00000400:00000010:0.0:1788972514.253867:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084dd0.
00000100:00000200:0.0:1788972514.253874:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb5880 x1875872179099904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788972514.253884:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb5880 x1875872179099904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972530 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788972514.253930:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548000.
02000000:00000010:0.0:1788972514.253934:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49714100.
00000100:00000010:0.0:1788972514.253939:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb5880.
00000400:00000200:3.0:1788972514.258192:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.258202:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972514.258209:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.258214:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.258218:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.258223:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e11880
00000400:00000200:3.0:1788972514.258228:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972514.258247:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1d15 [1] + 224
00000400:00000200:3.0:1788972514.258260:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972514.258265:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972514.258278:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb4000 x1875872179099776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972530 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972514.258821:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10.
00000400:00000200:1.0:1788972514.258828:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880
00000400:00000010:1.0:1788972514.258830:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880.
00000100:00000200:1.0:1788972514.258838:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb4000 x1875872179099776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972514.258859:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb4000 x1875872179099776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972530 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788972514.258886:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d68e00.
02000000:00000010:1.0:1788972514.258892:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49714300.
00000100:00000010:1.0:1788972514.258897:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4000.
00000100:00000010:3.0:1788972514.951373:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449698a0.
00000100:00000010:2.0:1788972514.953395:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969d20.
00000100:00000010:3.0:1788972519.369296:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7100.
02000000:00000010:3.0:1788972519.369437:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0da00.
02000000:00000010:2.0:1788972519.369497:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3a00.
00000400:00000200:2.0:1788972519.369507:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788972519.369511:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160.
00000400:00000010:2.0:1788972519.369518:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788972519.369532:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179100032, portal 25
00000400:00000010:2.0:1788972519.369537:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0110.
00000100:00000200:2.0:1788972519.369540:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179100032, offset 0
00000400:00000200:2.0:1788972519.369549:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788972519.369557:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972519.369565:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788972519.369571:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972519.369575:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788972519.369579:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972519.369582:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788972519.369585:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [568:568:256:4294967295] 192.168.203.124@tcp LPNI seq info [568:568:8:4294967295]
00000400:00000200:2.0:1788972519.369589:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972519.369598:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972519.369604:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788972519.369609:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1700.
00000800:00000200:2.0:1788972519.369615:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788972519.369622:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788972519.369627:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972519.369641:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788972519.369645:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0.
00000100:00000010:2.0:1788972519.369655:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7de0.
00000100:00000010:3.0:1788972519.369687:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5c00.
02000000:00000010:3.0:1788972519.369693:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0df00.
02000000:00000010:2.0:1788972519.369827:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2000.
00000400:00000200:2.0:1788972519.369833:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788972519.369835:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3108.
00000400:00000010:2.0:1788972519.369839:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0bb0.
00000100:00000200:2.0:1788972519.369842:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179100160, portal 10
00000400:00000010:2.0:1788972519.369848:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660.
00000100:00000200:2.0:1788972519.369850:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179100160, offset 0
00000400:00000200:2.0:1788972519.369854:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788972519.369858:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972519.369861:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788972519.369869:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972519.369872:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788972519.369874:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972519.369876:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788972519.369879:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [569:569:255:4294967295] 192.168.203.124@tcp LPNI seq info [569:569:7:4294967295]
00000400:00000200:2.0:1788972519.369883:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788972519.369889:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972519.369893:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788972519.369897:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100.
00000800:00000200:2.0:1788972519.369901:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788972519.369906:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788972519.369908:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972519.369914:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788972519.374883:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972519.374892:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1700.
00000400:00000200:2.0:1788972519.374898:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972519.374906:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788972519.374913:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0110
00000400:00000010:2.0:1788972519.374915:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0110.
00000100:00000200:2.0:1788972519.374935:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c7100 x1875872179100032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788972519.375020:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972519.375023:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100.
00000400:00000200:2.0:1788972519.375027:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972519.375030:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788972519.375034:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660
00000400:00000010:2.0:1788972519.375036:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660.
00000100:00000200:2.0:1788972519.375042:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb434c5c00 x1875872179100160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972519.379199:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379208:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972519.379216:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379221:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379227:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379231:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e11980
00000400:00000200:3.0:1788972519.379237:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379243:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1d1d [1] + 224
00000400:00000200:3.0:1788972519.379255:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972519.379260:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972519.379275:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c7100 x1875872179100032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972535 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972519.379335:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379339:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788972519.379344:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379347:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379350:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379353:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e11a00
00000400:00000200:3.0:1788972519.379356:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788972519.379360:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1d2d [1] + 224
00000400:00000200:3.0:1788972519.379380:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972519.379384:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788972519.379391:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb434c5c00 x1875872179100160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972535 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972519.379625:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3108.
00000400:00000200:2.0:1788972519.379636:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0bb0
00000400:00000010:2.0:1788972519.379639:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0bb0.
00000100:00000200:2.0:1788972519.379646:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c5c00 x1875872179100160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972535 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972519.379656:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c5c00 x1875872179100160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972535 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972519.379684:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160.
00000400:00000200:2.0:1788972519.379687:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8
00000400:00000010:2.0:1788972519.379689:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8.
00000100:00000200:2.0:1788972519.379694:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb434c7100 x1875872179100032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972535 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972519.379702:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb434c7100 x1875872179100032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972535 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972519.379727:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2000.
02000000:00000010:2.0:1788972519.379734:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0df00.
00000100:00000010:2.0:1788972519.379738:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5c00.
02000000:00000010:2.0:1788972519.379742:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3a00.
02000000:00000010:2.0:1788972519.379745:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0da00.
00000100:00000010:2.0:1788972519.379747:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7100.
00000100:00000010:2.0:1788972520.074517:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0.
00000100:00000010:3.0:1788972520.075887:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7de0.
00000001:02000400:0.0:1788972520.542828:0:120210:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 402: Destination net rule should not panic ========================================================== 12:48:39 (1788972519)
00000001:02000400:0.0:1788972520.778354:0:120224:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:0.0:1788972522.047017:0:120350:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_402: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.