00000100:00000010:3.0F:1790701379.567492:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908dae300.
02000000:00000010:3.0:1790701379.567524:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656600.
02000000:00000010:2.0F:1790701379.567615:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910d2a000.
00000400:00000200:2.0:1790701379.567631:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701379.567635:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823948.
00000400:00000010:2.0:1790701379.567642:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeee0.
00000100:00000200:2.0:1790701379.567650:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520512, portal 25
00000400:00000010:2.0:1790701379.567654:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee6e8.
00000100:00000200:2.0:1790701379.567658:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323520512, offset 0
00000400:00000200:2.0:1790701379.567668:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701379.567676:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701379.567684:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701379.567689:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701379.567693:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701379.567697:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701379.567700:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:2.0:1790701379.567703:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [76:76:256:4294967295] 192.168.206.116@tcp LPNI seq info [76:76:8:4294967295]
00000400:00000200:2.0:1790701379.567708:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701379.567717:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1790701379.567723:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:2.0:1790701379.567728:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1100.
00000800:00000200:2.0:1790701379.567734:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:2.0:1790701379.567754:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:2.0:1790701379.567758:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1790701379.567776:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:2.0:1790701379.567781:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922419060.
00000100:00000010:2.0:1790701379.567793:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922419d80.
00000100:00000010:3.0:1790701379.568005:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c908daf100.
02000000:00000010:3.0:1790701379.568013:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c92d656300.
00000800:00000200:1.0F:1790701379.568148:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701379.568190:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1100.
00000400:00000200:1.0:1790701379.568195:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701379.568201:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701379.568207:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee6e8
00000400:00000010:1.0:1790701379.568209:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee6e8.
00000100:00000200:1.0:1790701379.568218:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c908dae300 x1877686323520512/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701379.570647:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910d2b600.
00000400:00000200:2.0:1790701379.570663:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701379.570668:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823f20.
00000400:00000010:2.0:1790701379.570675:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeb28.
00000100:00000200:2.0:1790701379.570682:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520640, portal 10
00000400:00000010:2.0:1790701379.570689:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee220.
00000100:00000200:2.0:1790701379.570693:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323520640, offset 0
00000400:00000200:2.0:1790701379.570716:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701379.570724:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701379.570731:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701379.570736:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701379.570826:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701379.570830:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701379.570833:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:2.0:1790701379.570836:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [77:77:256:4294967295] 192.168.206.116@tcp LPNI seq info [77:77:8:4294967295]
00000400:00000200:2.0:1790701379.570842:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701379.570925:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1790701379.570933:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:2.0:1790701379.570939:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1e00.
00000800:00000200:2.0:1790701379.570945:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:2.0:1790701379.570953:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:2.0:1790701379.570957:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1790701379.571083:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701379.583359:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701379.583368:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:1.0:1790701379.583376:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701379.583381:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701379.583395:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701379.583399:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502400
00000400:00000200:1.0:1790701379.583405:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701379.583411:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x591 [1] + 224
00000400:00000200:1.0:1790701379.583432:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701379.583437:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:1.0:1790701379.583451:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c908dae300 x1877686323520512/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701395 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701379.583539:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701379.583543:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1e00.
00000400:00000200:1.0:1790701379.583547:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701379.583551:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701379.583555:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee220
00000400:00000010:1.0:1790701379.583558:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee220.
00000100:00000200:1.0:1790701379.583564:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c908daf100 x1877686323520640/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701395 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701379.583908:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823948.
00000400:00000200:3.0:1790701379.583913:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeee0
00000400:00000010:3.0:1790701379.583915:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeee0.
00000100:00000200:3.0:1790701379.583923:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c908dae300 x1877686323520512/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701395 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701379.583932:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c908dae300 x1877686323520512/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701395 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701379.583956:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910d2a000.
02000000:00000010:3.0:1790701379.583962:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656600.
00000100:00000010:3.0:1790701379.583966:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908dae300.
00000400:00000200:0.0F:1790701379.595638:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701379.595660:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701379.595669:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701379.595675:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701379.595680:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701379.595683:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502480
00000400:00000200:0.0:1790701379.595689:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701379.595695:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x599 [1] + 224
00000400:00000200:0.0:1790701379.595720:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701379.595725:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701379.595739:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c908daf100 x1877686323520640/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701395 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701379.595836:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823f20.
00000400:00000200:3.0:1790701379.595841:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeb28
00000400:00000010:3.0:1790701379.595844:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeb28.
00000100:00000200:3.0:1790701379.595851:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c908daf100 x1877686323520640/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701395 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701379.595860:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c908daf100 x1877686323520640/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701395 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701379.595890:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910d2b600.
02000000:00000010:3.0:1790701379.595896:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c92d656300.
00000100:00000010:3.0:1790701379.595910:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c908daf100.
00000100:00000010:1.0:1790701380.524749:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922419d80.
00000100:00000010:0.0:1790701380.525068:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922419060.
00000100:00000010:2.0:1790701384.684403:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96300.
02000000:00000010:2.0:1790701384.684427:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1c00.
00000100:00000010:2.0:1790701384.684465:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b97800.
02000000:00000010:2.0:1790701384.684471:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1a00.
02000000:00000010:2.0:1790701384.684505:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910d2ac00.
00000400:00000200:2.0:1790701384.684515:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701384.684519:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823b00.
00000400:00000010:2.0:1790701384.684525:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee908.
00000100:00000200:2.0:1790701384.684533:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520896, portal 10
00000400:00000010:2.0:1790701384.684540:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeec38.
00000100:00000200:2.0:1790701384.684544:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323520896, offset 0
00000400:00000200:2.0:1790701384.684552:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701384.684560:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701384.684568:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701384.684574:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701384.684591:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701384.684595:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701384.684598:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:2.0:1790701384.684602:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [78:78:256:4294967295] 192.168.206.116@tcp LPNI seq info [78:78:8:4294967295]
00000400:00000200:2.0:1790701384.684606:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701384.684617:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1790701384.684623:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:2.0:1790701384.684627:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1800.
00000800:00000200:2.0:1790701384.684634:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:2.0:1790701384.684641:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:2.0:1790701384.684645:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1790701384.684673:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:2.0:1790701384.684679:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922419a20.
00000100:00000010:2.0:1790701384.684700:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922419660.
02000000:00000010:2.0:1790701384.684722:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910d2a000.
00000400:00000200:2.0:1790701384.684726:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701384.684729:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823c08.
00000400:00000010:2.0:1790701384.684733:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee5d8.
00000100:00000200:2.0:1790701384.684736:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323520768, portal 25
00000400:00000010:2.0:1790701384.684739:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee3b8.
00000100:00000200:2.0:1790701384.684741:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323520768, offset 0
00000400:00000200:2.0:1790701384.684745:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701384.684749:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701384.684752:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701384.684754:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701384.684757:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701384.684758:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701384.684760:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:2.0:1790701384.684763:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [79:79:255:4294967295] 192.168.206.116@tcp LPNI seq info [79:79:7:4294967295]
00000400:00000200:2.0:1790701384.684766:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701384.684771:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1790701384.684781:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:2.0:1790701384.684784:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1400.
00000800:00000200:2.0:1790701384.684787:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:2.0:1790701384.684791:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:2.0:1790701384.684793:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1790701384.684797:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:1.0:1790701384.684832:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701384.684836:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1800.
00000400:00000200:1.0:1790701384.684842:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701384.684852:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701384.684856:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeec38
00000400:00000010:1.0:1790701384.684859:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeec38.
00000100:00000200:1.0:1790701384.684867:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b97800 x1877686323520896/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701384.684902:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701384.684904:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1400.
00000400:00000200:1.0:1790701384.684907:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701384.684910:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701384.684913:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee3b8
00000400:00000010:1.0:1790701384.684914:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee3b8.
00000100:00000200:1.0:1790701384.684919:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b96300 x1877686323520768/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701400 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701384.688602:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688611:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701384.688619:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688624:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688629:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688633:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502580
00000400:00000200:0.0:1790701384.688637:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688644:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x5a1 [1] + 224
00000400:00000200:0.0:1790701384.688668:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701384.688673:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701384.688686:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b97800 x1877686323520896/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701400 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701384.688720:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688723:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701384.688727:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688730:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688733:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688736:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502500
00000400:00000200:0.0:1790701384.688750:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701384.688755:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x5a9 [1] + 224
00000400:00000200:0.0:1790701384.688765:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701384.688768:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701384.688775:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b96300 x1877686323520768/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701400 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701384.689193:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823c08.
00000400:00000200:3.0:1790701384.689199:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee5d8
00000400:00000010:3.0:1790701384.689201:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee5d8.
00000100:00000200:3.0:1790701384.689208:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b96300 x1877686323520768/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701400 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701384.689216:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b96300 x1877686323520768/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701400 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701384.689260:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823b00.
00000400:00000200:3.0:1790701384.689263:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee908
00000400:00000010:3.0:1790701384.689264:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee908.
00000100:00000200:3.0:1790701384.689269:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b97800 x1877686323520896/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701400 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701384.689276:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b97800 x1877686323520896/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701400 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701384.689296:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910d2a000.
02000000:00000010:3.0:1790701384.689301:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1c00.
00000100:00000010:3.0:1790701384.689306:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96300.
02000000:00000010:3.0:1790701384.689309:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910d2ac00.
02000000:00000010:3.0:1790701384.689323:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1a00.
00000100:00000010:3.0:1790701384.689326:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b97800.
00000100:00000010:0.0:1790701385.645213:0:124073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922419a20.
00000100:00000010:1.0:1790701385.645508:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922419660.
00000001:02000400:3.0:1790701386.613115:0:126904:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 502: Verify lnetctl peer set --health (MR) ========================================================== 13:03:04 (1790701384)
00000001:02000400:1.0:1790701386.722612:0:126910:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:1.0:1790701388.037652:0:127045:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_502: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 215 lines, 215 kept, 0 dropped, 0 bad.