00000100:00000010:0.0F:1790701364.204558:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a21500.
02000000:00000010:0.0:1790701364.204607:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3b00.
00000100:00000010:0.0:1790701364.204656:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a23800.
02000000:00000010:0.0:1790701364.204662:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3f00.
02000000:00000010:1.0F:1790701364.204761:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf7a00.
00000400:00000200:1.0:1790701364.204788:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701364.204793:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69d10.
00000400:00000010:1.0:1790701364.204803:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775be58.
00000100:00000200:1.0:1790701364.204812:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519744, portal 25
00000400:00000010:1.0:1790701364.204819:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775ba18.
00000100:00000200:1.0:1790701364.204823:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323519744, offset 0
00000400:00000200:1.0:1790701364.204846:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701364.204858:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701364.204869:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701364.204877:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701364.204883:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701364.204888:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701364.204891:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701364.204896:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [70:70:256:4294967295] 192.168.206.116@tcp LPNI seq info [70:70:8:4294967295]
00000400:00000200:1.0:1790701364.204901:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701364.204912:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701364.204919:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701364.204925:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075000.
00000800:00000200:1.0:1790701364.204934:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701364.204945:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701364.204951:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701364.204976:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701364.205047:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac2a0.
00000100:00000010:1.0:1790701364.205063:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacd80.
02000000:00000010:1.0:1790701364.205088:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6200.
00000400:00000200:1.0:1790701364.205094:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701364.205096:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580.
00000400:00000010:1.0:1790701364.205101:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701364.205104:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323519872, portal 10
00000400:00000010:1.0:1790701364.205112:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b3b8.
00000100:00000200:1.0:1790701364.205114:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323519872, offset 0
00000400:00000200:1.0:1790701364.205126:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701364.205130:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701364.205133:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701364.205136:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701364.205139:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701364.205141:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701364.205144:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701364.205146:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [71:71:255:4294967295] 192.168.206.116@tcp LPNI seq info [71:71:7:4294967295]
00000400:00000200:1.0:1790701364.205151:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701364.205198:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701364.205203:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701364.205207:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075b00.
00000800:00000200:1.0:1790701364.205211:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701364.205215:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701364.205218:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701364.205225:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:0.0:1790701364.205875:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701364.205882:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075000.
00000400:00000200:0.0:1790701364.205892:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701364.205899:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701364.205907:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775ba18
00000400:00000010:0.0:1790701364.205909:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775ba18.
00000100:00000200:0.0:1790701364.205923:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c919a21500 x1877686323519744/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1790701364.207668:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701364.207679:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075b00.
00000400:00000200:0.0:1790701364.207689:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701364.207699:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701364.207706:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b3b8
00000400:00000010:0.0:1790701364.207709:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b3b8.
00000100:00000200:0.0:1790701364.207719:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c919a23800 x1877686323519872/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701380 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701364.214722:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.214735:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701364.214747:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.214753:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.214760:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.214764:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502180
00000400:00000200:0.0:1790701364.214771:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.214778:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x541 [1] + 224
00000400:00000200:0.0:1790701364.214793:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701364.214799:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701364.214819:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c919a23800 x1877686323519872/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701380 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701364.215740:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.215748:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701364.215759:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.215763:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.215767:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.215771:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502100
00000400:00000200:0.0:1790701364.215776:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701364.215782:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x531 [1] + 224
00000400:00000200:0.0:1790701364.215799:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701364.215803:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701364.215815:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c919a21500 x1877686323519744/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701380 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701364.216008:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580.
00000400:00000200:1.0:1790701364.216014:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908
00000400:00000010:1.0:1790701364.216017:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701364.216026:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c919a23800 x1877686323519872/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701380 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701364.216035:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c919a23800 x1877686323519872/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701380 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701364.216076:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69d10.
00000400:00000200:1.0:1790701364.216079:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775be58
00000400:00000010:1.0:1790701364.216081:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775be58.
00000100:00000200:1.0:1790701364.216085:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c919a21500 x1877686323519744/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701380 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701364.216094:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c919a21500 x1877686323519744/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701380 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1790701364.216130:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6200.
02000000:00000010:1.0:1790701364.216137:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3f00.
00000100:00000010:1.0:1790701364.216142:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a23800.
02000000:00000010:1.0:1790701364.216146:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf7a00.
02000000:00000010:1.0:1790701364.216149:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3b00.
00000100:00000010:1.0:1790701364.216171:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a21500.
00000100:00000010:0.0:1790701365.166702:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac2a0.
00000100:00000010:1.0:1790701365.166820:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacd80.
00000100:00000010:1.0:1790701369.324344:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300.
02000000:00000010:1.0:1790701369.324369:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075200.
00000100:00000010:1.0:1790701369.324413:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd0a80.
02000000:00000010:1.0:1790701369.324419:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075d00.
02000000:00000010:1.0:1790701369.324452:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6c00.
00000400:00000200:1.0:1790701369.324462:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701369.324467:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69d10.
00000400:00000010:1.0:1790701369.324473:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775be58.
00000100:00000200:1.0:1790701369.324480:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520128, portal 10
00000400:00000010:1.0:1790701369.324488:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701369.324491:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323520128, offset 0
00000400:00000200:1.0:1790701369.324499:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701369.324506:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.324514:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701369.324518:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701369.324522:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701369.324526:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701369.324529:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701369.324533:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [72:72:256:4294967295] 192.168.206.116@tcp LPNI seq info [72:72:8:4294967295]
00000400:00000200:1.0:1790701369.324538:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.324547:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701369.324552:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701369.324557:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075800.
00000800:00000200:1.0:1790701369.324576:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701369.324583:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701369.324588:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701369.324603:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701369.324609:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacd80.
00000100:00000010:1.0:1790701369.324625:0:121246:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac060.
02000000:00000010:1.0:1790701369.324645:0:121246:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c908bf6800.
00000400:00000200:1.0:1790701369.324649:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701369.324651:0:121246:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580.
00000400:00000010:1.0:1790701369.324655:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0.
00000100:00000200:1.0:1790701369.324658:0:121246:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520000, portal 25
00000400:00000010:1.0:1790701369.324662:0:121246:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220.
00000100:00000200:1.0:1790701369.324664:0:121246:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323520000, offset 0
00000400:00000200:1.0:1790701369.324669:0:121246:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701369.324672:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.324676:0:121246:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701369.324679:0:121246:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701369.324681:0:121246:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701369.324684:0:121246:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701369.324686:0:121246:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701369.324689:0:121246:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [73:73:255:4294967295] 192.168.206.116@tcp LPNI seq info [73:73:7:4294967295]
00000400:00000200:1.0:1790701369.324693:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.324699:0:121246:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701369.324704:0:121246:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701369.324707:0:121246:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075500.
00000800:00000200:1.0:1790701369.324711:0:121246:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701369.324715:0:121246:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701369.324717:0:121246:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701369.324721:0:121246:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:1.0:1790701369.324764:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701369.324774:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075800.
00000400:00000200:1.0:1790701369.324780:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701369.324785:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701369.324790:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908
00000400:00000010:1.0:1790701369.324793:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701369.324801:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd0a80 x1877686323520128/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701369.324874:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701369.324877:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075500.
00000400:00000200:1.0:1790701369.324881:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701369.324885:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701369.324887:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220
00000400:00000010:1.0:1790701369.324889:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220.
00000100:00000200:1.0:1790701369.324894:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd2300 x1877686323520000/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701385 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701369.331221:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701369.331231:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701369.331239:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701369.331245:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701369.331250:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701369.331255:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502200
00000400:00000200:0.0:1790701369.331260:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701369.331266:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x561 [1] + 224
00000400:00000200:0.0:1790701369.331280:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701369.331284:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701369.331299:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd2300 x1877686323520000/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701385 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701369.331750:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580.
00000400:00000200:1.0:1790701369.331755:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0
00000400:00000010:1.0:1790701369.331758:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0.
00000100:00000200:1.0:1790701369.331765:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd2300 x1877686323520000/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701385 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701369.331774:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd2300 x1877686323520000/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701385 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1790701369.331800:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6800.
02000000:00000010:1.0:1790701369.331806:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075200.
00000100:00000010:1.0:1790701369.331809:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300.
00000400:00000200:1.0:1790701369.335501:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.335511:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:1.0:1790701369.335519:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.335525:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.335530:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.335534:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502280
00000400:00000200:1.0:1790701369.335553:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701369.335559:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x551 [1] + 224
00000400:00000200:1.0:1790701369.335575:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701369.335580:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:1.0:1790701369.335595:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd0a80 x1877686323520128/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701385 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1790701369.335717:0:121246:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69d10.
00000400:00000200:0.0:1790701369.335723:0:121246:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775be58
00000400:00000010:0.0:1790701369.335725:0:121246:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775be58.
00000100:00000200:0.0:1790701369.335738:0:121246:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd0a80 x1877686323520128/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701385 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1790701369.335748:0:121246:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd0a80 x1877686323520128/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701385 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1790701369.335781:0:121246:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c908bf6c00.
02000000:00000010:0.0:1790701369.335787:0:121246:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075d00.
00000100:00000010:0.0:1790701369.335791:0:121246:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd0a80.
00000100:00000010:1.0:1790701370.284283:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac060.
00000100:00000010:0.0:1790701370.284397:0:124073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacd80.
00000001:02000400:1.0:1790701372.654896:0:126192:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 501: Verify lnetctl peer set --health (SR) ========================================================== 13:02:51 (1790701371)
00000001:02000400:3.0F:1790701372.875466:0:126205:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:3.0:1790701374.221003:0:126332:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_501: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:1.0:1790701374.448452:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd2300.
02000000:00000010:1.0:1790701374.448470:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075c00.
00000100:00000010:1.0:1790701374.448495:0:37:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910bd1c00.
02000000:00000010:1.0:1790701374.448500:0:37:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c923075600.
02000000:00000010:1.0:1790701374.448526:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c923625c00.
00000400:00000200:1.0:1790701374.448536:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701374.448540:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580.
00000400:00000010:1.0:1790701374.448547:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0.
00000100:00000200:1.0:1790701374.448554:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520256, portal 25
00000400:00000010:1.0:1790701374.448559:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220.
00000100:00000200:1.0:1790701374.448561:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323520256, offset 0
00000400:00000200:1.0:1790701374.448699:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701374.448710:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.448718:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701374.448724:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701374.448729:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701374.448733:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701374.448737:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701374.448740:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [74:74:256:4294967295] 192.168.206.116@tcp LPNI seq info [74:74:8:4294967295]
00000400:00000200:1.0:1790701374.448745:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.448758:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701374.448775:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701374.448779:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075a00.
00000800:00000200:1.0:1790701374.448787:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701374.448796:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701374.448801:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701374.448816:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701374.448821:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac360.
00000100:00000010:1.0:1790701374.448833:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cacf00.
02000000:00000010:1.0:1790701374.448856:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c923624000.
00000400:00000200:1.0:1790701374.448861:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701374.448863:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318.
00000400:00000010:1.0:1790701374.448867:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701374.448871:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520384, portal 10
00000400:00000010:1.0:1790701374.448876:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0.
00000100:00000200:1.0:1790701374.448879:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323520384, offset 0
00000400:00000200:1.0:1790701374.448894:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701374.448899:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.448903:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701374.448906:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701374.448910:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701374.448912:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701374.448915:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701374.448918:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [75:75:255:4294967295] 192.168.206.116@tcp LPNI seq info [75:75:7:4294967295]
00000400:00000200:1.0:1790701374.448922:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.448928:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701374.448934:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701374.448937:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075300.
00000800:00000200:1.0:1790701374.448941:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701374.448946:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701374.448948:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701374.448961:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:1.0:1790701374.449095:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701374.449099:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075a00.
00000400:00000200:1.0:1790701374.449103:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701374.449108:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701374.449113:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220
00000400:00000010:1.0:1790701374.449115:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220.
00000100:00000200:1.0:1790701374.449124:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd2300 x1877686323520256/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701374.449214:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701374.449217:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075300.
00000400:00000200:1.0:1790701374.449220:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701374.449223:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701374.449226:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0
00000400:00000010:1.0:1790701374.449228:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0.
00000100:00000200:1.0:1790701374.449234:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910bd1c00 x1877686323520384/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701390 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1790701374.453476:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453486:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:1.0:1790701374.453493:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453498:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453502:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453506:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502300
00000400:00000200:1.0:1790701374.453511:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453516:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x571 [1] + 224
00000400:00000200:1.0:1790701374.453527:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701374.453532:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:1.0:1790701374.453547:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd2300 x1877686323520256/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701390 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1790701374.453818:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453822:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:1.0:1790701374.453839:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453841:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453844:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453847:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502380
00000400:00000200:1.0:1790701374.453851:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701374.453855:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x581 [1] + 224
00000400:00000200:1.0:1790701374.453865:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701374.453868:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:1.0:1790701374.453875:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910bd1c00 x1877686323520384/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701390 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701374.454089:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318.
00000400:00000200:1.0:1790701374.454094:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908
00000400:00000010:1.0:1790701374.454096:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701374.454103:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd1c00 x1877686323520384/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701390 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701374.454111:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd1c00 x1877686323520384/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701390 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701374.454139:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580.
00000400:00000200:1.0:1790701374.454141:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0
00000400:00000010:1.0:1790701374.454143:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0.
00000100:00000200:1.0:1790701374.454148:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910bd2300 x1877686323520256/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701390 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701374.454180:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910bd2300 x1877686323520256/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701390 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1790701374.454197:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c923624000.
02000000:00000010:1.0:1790701374.454203:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075600.
00000100:00000010:1.0:1790701374.454207:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd1c00.
02000000:00000010:1.0:1790701374.454211:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c923625c00.
02000000:00000010:1.0:1790701374.454213:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c923075c00.
00000100:00000010:1.0:1790701374.454222:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910bd2300.
00000100:00000010:1.0:1790701375.404659:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cacf00.
00000100:00000010:0.0:1790701375.405235:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac360.
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.