00000100:00000010:0.0F:1790701334.444283:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3a80.
00000100:00000010:1.0F:1790701334.448677:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3660.
00000100:00000010:2.0F:1790701338.604509:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94000.
02000000:00000010:2.0:1790701338.604545:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1e00.
00000100:00000010:2.0:1790701338.604598:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96300.
02000000:00000010:2.0:1790701338.604603:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1d00.
02000000:00000010:2.0:1790701338.604641:0:121248:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c9350ca400.
00000400:00000200:2.0:1790701338.604653:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701338.604657:0:121248:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823948.
00000400:00000010:2.0:1790701338.604666:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeee0.
00000100:00000200:2.0:1790701338.604674:0:121248:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518592, portal 10
00000400:00000010:2.0:1790701338.604683:0:121248:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee550.
00000100:00000200:2.0:1790701338.604687:0:121248:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323518592, offset 0
00000400:00000200:2.0:1790701338.604696:0:121248:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701338.604704:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701338.604712:0:121248:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701338.604717:0:121248:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701338.604721:0:121248:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701338.604724:0:121248:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701338.604727:0:121248:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
02000000:00000010:3.0F:1790701338.604729:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c91a043000.
00000400:00000200:2.0:1790701338.604731:0:121248:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [60:60:256:4294967295] 192.168.206.116@tcp LPNI seq info [60:60:8:4294967295]
00000400:00000200:2.0:1790701338.604735:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701338.604739:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:3.0:1790701338.604742:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318.
00000400:00000010:3.0:1790701338.604745:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198.
00000400:00000200:2.0:1790701338.604745:0:121248:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000100:00000200:3.0:1790701338.604748:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518464, portal 25
00000400:00000010:3.0:1790701338.604751:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35660.
00000800:00000200:2.0:1790701338.604751:0:121248:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000100:00000200:3.0:1790701338.604754:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323518464, offset 0
00000800:00000010:2.0:1790701338.604756:0:121248:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1900.
00000400:00000200:3.0:1790701338.604758:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:3.0:1790701338.604762:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:2.0:1790701338.604763:0:121248:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000400:00000200:3.0:1790701338.604765:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:3.0:1790701338.604767:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701338.604770:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000800:00000200:2.0:1790701338.604770:0:121248:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000400:00000200:3.0:1790701338.604772:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701338.604774:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000800:00000200:2.0:1790701338.604774:0:121248:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701338.604777:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [61:61:255:4294967295] 192.168.206.116@tcp LPNI seq info [61:61:7:4294967295]
00000400:00000200:3.0:1790701338.604781:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701338.604786:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701338.604791:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:3.0:1790701338.604794:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656a00.
00000800:00000200:3.0:1790701338.604798:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000400:00000200:2.0:1790701338.604799:0:121248:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:3.0:1790701338.604801:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:3.0:1790701338.604804:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656a00 type 1, nob 320 niov 1 nkiov 1
00000100:00000010:2.0:1790701338.604804:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c90a1b9360.
00000400:00000200:3.0:1790701338.604807:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:2.0:1790701338.604818:0:121248:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c90a1b9a80.
00000800:00000200:0.0:1790701338.604889:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701338.604907:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1900.
00000400:00000200:0.0:1790701338.604912:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701338.604917:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701338.604922:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee550
00000400:00000010:0.0:1790701338.604924:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee550.
00000100:00000200:0.0:1790701338.604932:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b96300 x1877686323518592/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1790701338.605008:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701338.605011:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656a00.
00000400:00000200:0.0:1790701338.605014:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701338.605018:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701338.605020:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35660
00000400:00000010:0.0:1790701338.605022:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35660.
00000100:00000200:0.0:1790701338.605026:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b94000 x1877686323518464/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701354 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701338.613298:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613309:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701338.613320:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613325:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613331:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613335:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501c00
00000400:00000200:0.0:1790701338.613340:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613346:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x4b9 [1] + 224
00000400:00000200:0.0:1790701338.613358:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701338.613362:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701338.613376:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b94000 x1877686323518464/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701354 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701338.613418:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613422:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701338.613426:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613443:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613447:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613451:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501c80
00000400:00000200:0.0:1790701338.613468:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701338.613473:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x4b1 [1] + 224
00000400:00000200:0.0:1790701338.613529:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701338.613533:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701338.613540:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b96300 x1877686323518592/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701354 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701338.613697:0:121248:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823948.
00000400:00000200:2.0:1790701338.613702:0:121248:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeee0
00000400:00000010:2.0:1790701338.613704:0:121248:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeee0.
00000100:00000200:2.0:1790701338.613718:0:121248:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b96300 x1877686323518592/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701354 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701338.613726:0:121248:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b96300 x1877686323518592/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701354 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701338.613756:0:121248:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c9350ca400.
02000000:00000010:2.0:1790701338.613762:0:121248:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1d00.
00000100:00000010:2.0:1790701338.613767:0:121248:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96300.
00000400:00000010:3.0:1790701338.617108:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318.
00000400:00000200:3.0:1790701338.617139:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198
00000400:00000010:3.0:1790701338.617142:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198.
00000100:00000200:3.0:1790701338.617197:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b94000 x1877686323518464/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701354 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701338.617210:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b94000 x1877686323518464/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701354 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701338.617239:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c91a043000.
02000000:00000010:3.0:1790701338.617263:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1e00.
00000100:00000010:3.0:1790701338.617268:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94000.
00000100:00000010:0.0:1790701339.564319:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c90a1b9360.
00000100:00000010:1.0:1790701339.565358:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c90a1b9a80.
00000100:00000010:2.0:1790701343.732774:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b96300.
02000000:00000010:2.0:1790701343.732794:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1a00.
00000100:00000010:2.0:1790701343.732825:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b97800.
02000000:00000010:2.0:1790701343.732830:0:8:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1100.
02000000:00000010:2.0:1790701343.732868:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924003400.
00000400:00000200:2.0:1790701343.732877:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701343.732880:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823948.
00000400:00000010:2.0:1790701343.732887:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeeee0.
00000100:00000200:2.0:1790701343.732894:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518720, portal 25
00000400:00000010:2.0:1790701343.732900:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee550.
00000100:00000200:2.0:1790701343.732903:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323518720, offset 0
00000400:00000200:2.0:1790701343.732912:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701343.732920:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701343.732927:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701343.732932:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701343.732936:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701343.732939:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701343.732942:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:2.0:1790701343.732946:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [62:62:256:4294967295] 192.168.206.116@tcp LPNI seq info [62:62:8:4294967295]
00000400:00000200:2.0:1790701343.732950:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701343.732958:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1790701343.732963:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:2.0:1790701343.732968:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1400.
00000800:00000200:2.0:1790701343.732979:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:2.0:1790701343.732986:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:2.0:1790701343.732990:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1790701343.733004:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:2.0:1790701343.733009:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c90a1b9360.
00000100:00000010:2.0:1790701343.733020:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c90a1b9cc0.
02000000:00000010:2.0:1790701343.733039:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c924003800.
00000400:00000200:2.0:1790701343.733043:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:2.0:1790701343.733046:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c918823f20.
00000400:00000010:2.0:1790701343.733050:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eee6e8.
00000100:00000200:2.0:1790701343.733053:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323518848, portal 10
00000400:00000010:2.0:1790701343.733058:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c920eeee58.
00000100:00000200:2.0:1790701343.733060:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323518848, offset 0
00000400:00000200:2.0:1790701343.733064:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:2.0:1790701343.733068:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701343.733072:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:2.0:1790701343.733074:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1790701343.733076:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:2.0:1790701343.733078:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1790701343.733080:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:2.0:1790701343.733083:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [63:63:255:4294967295] 192.168.206.116@tcp LPNI seq info [63:63:7:4294967295]
00000400:00000200:2.0:1790701343.733086:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:2.0:1790701343.733092:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1790701343.733096:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:2.0:1790701343.733099:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c9351e1d00.
00000800:00000200:2.0:1790701343.733103:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:2.0:1790701343.733107:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:2.0:1790701343.733110:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c9351e1d00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1790701343.733115:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:0.0:1790701343.735094:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701343.735102:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1400.
00000400:00000200:0.0:1790701343.735108:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701343.735116:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701343.735123:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee550
00000400:00000010:0.0:1790701343.735125:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee550.
00000100:00000200:0.0:1790701343.735137:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b96300 x1877686323518720/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1790701343.735240:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701343.735243:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c9351e1d00.
00000400:00000200:0.0:1790701343.735246:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701343.735250:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701343.735253:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeee58
00000400:00000010:0.0:1790701343.735255:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeee58.
00000100:00000200:0.0:1790701343.735260:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b97800 x1877686323518848/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701359 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701343.739356:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739367:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701343.739375:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739380:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739385:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739395:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea501d80
00000400:00000200:0.0:1790701343.739401:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739406:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x4c9 [1] + 224
00000400:00000200:0.0:1790701343.739418:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701343.739422:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701343.739437:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b97800 x1877686323518848/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701359 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701343.739473:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739477:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701343.739481:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739484:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739487:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739490:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea501d00
00000400:00000200:0.0:1790701343.739493:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701343.739497:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x4c1 [1] + 224
00000400:00000200:0.0:1790701343.739505:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701343.739509:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701343.739515:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b96300 x1877686323518720/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701359 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701343.739583:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823f20.
00000400:00000200:2.0:1790701343.739588:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eee6e8
00000400:00000010:2.0:1790701343.739596:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eee6e8.
00000100:00000200:2.0:1790701343.739602:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b97800 x1877686323518848/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701359 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701343.739611:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b97800 x1877686323518848/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701359 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701343.739642:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c918823948.
00000400:00000200:2.0:1790701343.739645:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c920eeeee0
00000400:00000010:2.0:1790701343.739647:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c920eeeee0.
00000100:00000200:2.0:1790701343.739651:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b96300 x1877686323518720/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701359 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701343.739658:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b96300 x1877686323518720/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701359 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701343.739672:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924003800.
02000000:00000010:2.0:1790701343.739679:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1100.
00000100:00000010:2.0:1790701343.739683:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b97800.
02000000:00000010:2.0:1790701343.739687:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c924003400.
02000000:00000010:2.0:1790701343.739689:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1a00.
00000100:00000010:2.0:1790701343.739690:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b96300.
00000001:02000400:3.0:1790701344.642273:0:124766:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 476: lnetctl set latency_stats toggles capture ========================================================== 13:02:23 (1790701343)
00000100:00000010:0.0:1790701344.686577:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c90a1b9360.
00000100:00000010:1.0:1790701344.687313:0:72075:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c90a1b9cc0.
00000001:02000400:0.0:1790701344.731979:0:124772:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:2.0:1790701346.334358:0:124906:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_476: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 217 lines, 217 kept, 0 dropped, 0 bad.