00000100:00000010:2.0F:1790701389.804375:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b94e00.
02000000:00000010:2.0:1790701389.804401:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1d00.
00000100:00000010:2.0:1790701389.804434:0:36:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b97480.
02000000:00000010:2.0:1790701389.804439:0:36:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1300.
02000000:00000010:3.0F:1790701389.804461:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbe600.
00000400:00000200:3.0:1790701389.804476:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:3.0:1790701389.804481:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318.
00000400:00000010:3.0:1790701389.804487:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198.
00000100:00000200:3.0:1790701389.804493:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521024, portal 25
00000400:00000010:3.0:1790701389.804498:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35880.
00000100:00000200:3.0:1790701389.804501:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323521024, offset 0
00000400:00000200:3.0:1790701389.804510:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:3.0:1790701389.804517:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701389.804524:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:3.0:1790701389.804529:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701389.804532:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:3.0:1790701389.804536:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701389.804538:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:3.0:1790701389.804542:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [80:80:256:4294967295] 192.168.206.116@tcp LPNI seq info [80:80:8:4294967295]
00000400:00000200:3.0:1790701389.804546:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701389.804555:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701389.804561:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:3.0:1790701389.804566:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656e00.
00000800:00000200:3.0:1790701389.804572:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:3.0:1790701389.804581:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:3.0:1790701389.804585:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701389.804599:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:3.0:1790701389.804604:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3a80.
00000100:00000010:3.0:1790701389.804616:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3660.
02000000:00000010:3.0:1790701389.804637:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbe000.
00000400:00000200:3.0:1790701389.804641:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:3.0:1790701389.804643:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d90b0.
00000400:00000010:3.0:1790701389.804647:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35088.
00000100:00000200:3.0:1790701389.804650:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521152, portal 10
00000400:00000010:3.0:1790701389.804655:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d355d8.
00000100:00000200:3.0:1790701389.804658:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323521152, offset 0
00000400:00000200:3.0:1790701389.804668:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:3.0:1790701389.804671:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701389.804675:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:3.0:1790701389.804677:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701389.804680:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:3.0:1790701389.804682:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701389.804685:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:3.0:1790701389.804688:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [81:81:255:4294967295] 192.168.206.116@tcp LPNI seq info [81:81:7:4294967295]
00000400:00000200:3.0:1790701389.804692:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701389.804698:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701389.804702:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:3.0:1790701389.804706:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656d00.
00000800:00000200:3.0:1790701389.804710:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:3.0:1790701389.804715:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:3.0:1790701389.804717:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656d00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701389.804723:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:0.0F:1790701389.808660:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701389.808678:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656e00.
00000400:00000200:0.0:1790701389.808684:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701389.808693:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701389.808700:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35880
00000400:00000010:0.0:1790701389.808703:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35880.
00000100:00000200:0.0:1790701389.808714:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b94e00 x1877686323521024/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1790701389.808802:0:121237:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701389.808805:0:121237:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656d00.
00000400:00000200:0.0:1790701389.808808:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701389.808812:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701389.808815:0:121237:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d355d8
00000400:00000010:0.0:1790701389.808817:0:121237:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d355d8.
00000100:00000200:0.0:1790701389.808822:0:121237:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b97480 x1877686323521152/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701405 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701389.821011:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.821020:0:121238:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701389.821029:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.821034:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.821039:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.821042:0:121238:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502680
00000400:00000200:0.0:1790701389.821048:0:121238:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.821054:0:121238:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x5b9 [1] + 224
00000400:00000200:0.0:1790701389.821067:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701389.821071:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701389.821084:0:121238:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b97480 x1877686323521152/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701405 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701389.821324:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d90b0.
00000400:00000200:2.0:1790701389.821329:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35088
00000400:00000010:2.0:1790701389.821332:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35088.
00000100:00000200:2.0:1790701389.821339:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b97480 x1877686323521152/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701405 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701389.821347:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b97480 x1877686323521152/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701405 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701389.821378:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbe000.
02000000:00000010:2.0:1790701389.821384:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1300.
00000100:00000010:2.0:1790701389.821388:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b97480.
00000400:00000200:0.0:1790701389.828225:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.828235:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701389.828243:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.828255:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.828261:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.828265:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502600
00000400:00000200:0.0:1790701389.828270:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701389.828277:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x5b1 [1] + 224
00000400:00000200:0.0:1790701389.828304:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701389.828309:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701389.828322:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b94e00 x1877686323521024/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701405 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701389.829637:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318.
00000400:00000200:3.0:1790701389.829644:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198
00000400:00000010:3.0:1790701389.829646:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198.
00000100:00000200:3.0:1790701389.829657:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b94e00 x1877686323521024/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701405 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701389.829669:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b94e00 x1877686323521024/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701405 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701389.829695:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbe600.
02000000:00000010:3.0:1790701389.829701:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1d00.
00000100:00000010:3.0:1790701389.829706:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b94e00.
00000100:00000010:1.0F:1790701390.765222:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3660.
00000100:00000010:0.0:1790701390.765564:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3a80.
00000100:00000010:2.0:1790701394.924375:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b97480.
02000000:00000010:2.0:1790701394.924400:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1c00.
00000100:00000010:2.0:1790701394.924438:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c910b95180.
02000000:00000010:2.0:1790701394.924444:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c9351e1100.
02000000:00000010:3.0:1790701394.924532:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbfe00.
00000400:00000200:3.0:1790701394.924542:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:3.0:1790701394.924547:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9318.
00000400:00000010:3.0:1790701394.924553:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35198.
00000100:00000200:3.0:1790701394.924576:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521280, portal 25
00000400:00000010:3.0:1790701394.924581:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35a18.
00000100:00000200:3.0:1790701394.924584:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323521280, offset 0
00000400:00000200:3.0:1790701394.924593:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:3.0:1790701394.924600:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701394.924608:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:3.0:1790701394.924612:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701394.924617:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:3.0:1790701394.924620:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701394.924623:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:3.0:1790701394.924627:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [82:82:256:4294967295] 192.168.206.116@tcp LPNI seq info [82:82:8:4294967295]
00000400:00000200:3.0:1790701394.924632:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701394.924642:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701394.924647:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:3.0:1790701394.924653:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656a00.
00000800:00000200:3.0:1790701394.924659:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:3.0:1790701394.924665:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:3.0:1790701394.924670:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701394.924693:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:3.0:1790701394.924699:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e3d80.
00000100:00000010:3.0:1790701394.924710:0:121247:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c93b3e38a0.
02000000:00000010:3.0:1790701394.924731:0:121247:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c910fbf000.
00000400:00000200:3.0:1790701394.924735:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:3.0:1790701394.924737:0:121247:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c92d5d9f20.
00000400:00000010:3.0:1790701394.924741:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35dd0.
00000100:00000200:3.0:1790701394.924744:0:121247:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521408, portal 10
00000400:00000010:3.0:1790701394.924750:0:121247:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c910d35aa0.
00000100:00000200:3.0:1790701394.924752:0:121247:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323521408, offset 0
00000400:00000200:3.0:1790701394.924756:0:121247:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:3.0:1790701394.924760:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701394.924764:0:121247:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:3.0:1790701394.924772:0:121247:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701394.924775:0:121247:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:3.0:1790701394.924777:0:121247:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701394.924779:0:121247:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:3.0:1790701394.924781:0:121247:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [83:83:255:4294967295] 192.168.206.116@tcp LPNI seq info [83:83:7:4294967295]
00000400:00000200:3.0:1790701394.924785:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:3.0:1790701394.924791:0:121247:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701394.924796:0:121247:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:3.0:1790701394.924800:0:121247:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c92d656300.
00000800:00000200:3.0:1790701394.924804:0:121247:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:3.0:1790701394.924808:0:121247:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:3.0:1790701394.924810:0:121247:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c92d656300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701394.924816:0:121247:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:0.0:1790701394.924946:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701394.924951:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656a00.
00000400:00000200:0.0:1790701394.924957:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701394.924962:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701394.924967:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35a18
00000400:00000010:0.0:1790701394.924969:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35a18.
00000100:00000200:0.0:1790701394.924978:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b97480 x1877686323521280/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:0.0:1790701394.925043:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:0.0:1790701394.925046:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c92d656300.
00000400:00000200:0.0:1790701394.925048:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701394.925052:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:0.0:1790701394.925054:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35aa0
00000400:00000010:0.0:1790701394.925056:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35aa0.
00000100:00000200:0.0:1790701394.925061:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c910b95180 x1877686323521408/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701410 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701394.932513:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932522:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701394.932531:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932535:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932540:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932556:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502780
00000400:00000200:0.0:1790701394.932563:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932570:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x5c9 [1] + 224
00000400:00000200:0.0:1790701394.932582:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701394.932586:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701394.932599:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b95180 x1877686323521408/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701410 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701394.932678:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932681:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701394.932686:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932688:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932691:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932693:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502700
00000400:00000200:0.0:1790701394.932697:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701394.932701:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x5c1 [1] + 224
00000400:00000200:0.0:1790701394.932709:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701394.932712:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701394.932719:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c910b97480 x1877686323521280/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701410 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701394.932824:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9f20.
00000400:00000200:3.0:1790701394.933019:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35dd0
00000400:00000010:3.0:1790701394.933021:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35dd0.
00000100:00000200:3.0:1790701394.933029:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b95180 x1877686323521408/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701410 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701394.933038:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b95180 x1877686323521408/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701410 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701394.933066:0:121247:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c92d5d9318.
00000400:00000200:3.0:1790701394.933068:0:121247:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c910d35198
00000400:00000010:3.0:1790701394.933070:0:121247:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c910d35198.
00000100:00000200:3.0:1790701394.933074:0:121247:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c910b97480 x1877686323521280/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701410 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701394.933082:0:121247:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c910b97480 x1877686323521280/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701410 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701394.933095:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbf000.
02000000:00000010:3.0:1790701394.933100:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1100.
00000100:00000010:3.0:1790701394.933110:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b95180.
02000000:00000010:3.0:1790701394.933248:0:121247:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c910fbfe00.
02000000:00000010:3.0:1790701394.933252:0:121247:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c9351e1c00.
00000100:00000010:3.0:1790701394.933255:0:121247:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c910b97480.
00000100:00000010:0.0:1790701395.884522:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e3d80.
00000100:00000010:1.0:1790701395.884662:0:91773:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c93b3e38a0.
00000001:02000400:3.0:1790701398.507727:0:127616:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 525: keep reachable peer on local net delete ========================================================== 13:03:17 (1790701397)
00000001:02000400:2.0:1790701398.663688:0:127628:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000100:00000010:0.0:1790701400.046678:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20380.
02000000:00000010:0.0:1790701400.046695:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3a00.
02000000:00000010:1.0:1790701400.046758:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c94217e200.
00000400:00000200:1.0:1790701400.046768:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701400.046771:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69580.
00000400:00000010:1.0:1790701400.046778:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bdd0.
00000100:00000200:1.0:1790701400.046785:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521536, portal 25
00000400:00000010:1.0:1790701400.046790:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701400.046793:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686323521536, offset 0
00000400:00000200:1.0:1790701400.046800:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701400.046807:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701400.046814:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701400.046819:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701400.046822:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701400.046825:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701400.046828:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701400.046831:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [84:84:256:4294967295] 192.168.206.116@tcp LPNI seq info [84:84:8:4294967295]
00000400:00000200:1.0:1790701400.046836:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701400.046844:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701400.046850:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701400.046856:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075f00.
00000800:00000200:1.0:1790701400.046862:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701400.046868:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701400.046872:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701400.046886:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000100:00000010:1.0:1790701400.046892:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac720.
00000100:00000010:1.0:1790701400.046902:0:121245:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98c922cac5a0.
00000100:00000010:0.0:1790701400.046934:0:104617:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98c919a20000.
02000000:00000010:0.0:1790701400.046940:0:104617:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98c93bdb3900.
02000000:00000010:1.0:1790701400.047142:0:121245:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98c907a79200.
00000400:00000200:1.0:1790701400.047148:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000010:1.0:1790701400.047178:0:121245:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98c920e69318.
00000400:00000010:1.0:1790701400.047183:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775bee0.
00000100:00000200:1.0:1790701400.047186:0:121245:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686323521664, portal 10
00000400:00000010:1.0:1790701400.047299:0:121245:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98c90775b220.
00000100:00000200:1.0:1790701400.047302:0:121245:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686323521664, offset 0
00000400:00000200:1.0:1790701400.047308:0:121245:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.206.116@tcp
00000400:00000200:1.0:1790701400.047313:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701400.047317:0:121245:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.206.116@tcp local destination
00000400:00000200:1.0:1790701400.047319:0:121245:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1790701400.047322:0:121245:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.206.16@tcp
00000400:00000200:1.0:1790701400.047324:0:121245:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.206.16@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1790701400.047326:0:121245:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.206.116@tcp
00000400:00000200:1.0:1790701400.047329:0:121245:0:(lib-move.c:1738:lnet_handle_send()) 192.168.206.16@tcp NI seq info: [85:85:255:4294967295] 192.168.206.116@tcp LPNI seq info [85:85:7:4294967295]
00000400:00000200:1.0:1790701400.047333:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:1.0:1790701400.047338:0:121245:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp:<?>) -> 192.168.206.116@tcp(192.168.206.116@tcp:192.168.206.116@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1790701400.047343:0:121245:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.206.116@tcp
00000800:00000010:1.0:1790701400.047346:0:121245:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98c923075400.
00000800:00000200:1.0:1790701400.047350:0:121245:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98c907b1ca80] -> 12345-192.168.206.116@tcp (5)
00000800:00000200:1.0:1790701400.047355:0:121245:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.206.116@tcp ip 192.168.206.116:988
00000800:00000200:1.0:1790701400.047424:0:121245:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98c923075400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1790701400.047432:0:121245:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000800:00000200:1.0:1790701400.047545:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701400.047549:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075f00.
00000400:00000200:1.0:1790701400.047553:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701400.047558:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701400.047563:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b908
00000400:00000010:1.0:1790701400.047566:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b908.
00000100:00000200:1.0:1790701400.047576:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c919a20380 x1877686323521536/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:1.0:1790701400.047654:0:121238:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:1.0:1790701400.047657:0:121238:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98c923075400.
00000400:00000200:1.0:1790701400.047660:0:121238:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1790701400.047663:0:121238:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000400:00000200:1.0:1790701400.047666:0:121238:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775b220
00000400:00000010:1.0:1790701400.047674:0:121238:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775b220.
00000100:00000200:1.0:1790701400.047680:0:121238:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff98c919a20000 x1877686323521664/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701416 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701400.054648:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054658:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701400.054669:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054674:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054679:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054697:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 25 MB=0x6abbeea502800
00000400:00000200:0.0:1790701400.054703:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054710:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.206.116@tcp of length 224/224 into md 0x5d1 [1] + 224
00000400:00000200:0.0:1790701400.054721:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701400.054725:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701400.054739:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c919a20380 x1877686323521536/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701416 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:0.0:1790701400.054783:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054787:0:121237:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.206.16@tcp(192.168.206.16@tcp) <- 192.168.206.116@tcp : PUT - for me
00000400:00000200:0.0:1790701400.054791:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054794:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054797:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054811:0:121237:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.206.116@tcp of length 224 into portal 10 MB=0x6abbeea502880
00000400:00000200:0.0:1790701400.054815:0:121237:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.206.116@tcp to cpt 0
00000400:00000200:0.0:1790701400.054820:0:121237:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.206.116@tcp of length 224/224 into md 0x5e1 [1] + 224
00000400:00000200:0.0:1790701400.054831:0:121237:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:0.0:1790701400.054835:0:121237:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.206.16@tcp->192.168.206.116@tcp: PUT: OK
00000100:00000200:0.0:1790701400.054841:0:121237:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff98c919a20000 x1877686323521664/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701416 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701400.056177:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69318.
00000400:00000200:1.0:1790701400.056186:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bee0
00000400:00000010:1.0:1790701400.056199:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bee0.
00000100:00000200:1.0:1790701400.056210:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c919a20000 x1877686323521664/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701416 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701400.056220:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c919a20000 x1877686323521664/t0(0) o400->lustre-MDT0000-mdc-ffff98c907b8c000@192.168.206.116@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701416 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1790701400.056260:0:121245:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98c920e69580.
00000400:00000200:1.0:1790701400.056264:0:121245:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98c90775bdd0
00000400:00000010:1.0:1790701400.056265:0:121245:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98c90775bdd0.
00000100:00000200:1.0:1790701400.056270:0:121245:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff98c919a20380 x1877686323521536/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701416 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1790701400.056278:0:121245:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff98c919a20380 x1877686323521536/t0(0) o400->MGC192.168.206.116@tcp@192.168.206.116@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701416 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1790701400.056290:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c907a79200.
02000000:00000010:1.0:1790701400.056296:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3900.
00000100:00000010:1.0:1790701400.056300:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20000.
02000000:00000010:1.0:1790701400.056304:0:121245:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98c94217e200.
02000000:00000010:1.0:1790701400.056308:0:121245:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98c93bdb3a00.
00000100:00000010:1.0:1790701400.056310:0:121245:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98c919a20380.
00000001:02000400:0.0:1790701400.119975:0:127757:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_525: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:0.0:1790701401.004746:0:86991:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac720.
00000100:00000010:1.0:1790701401.005311:0:125063:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98c922cac5a0.
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.