00000100:00000010:1.0F:1788971981.769226:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800.
02000000:00000010:1.0:1788971981.769251:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51637a00.
00000100:00000010:1.0:1788971981.769279:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80.
02000000:00000010:1.0:1788971981.769285:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51637100.
02000000:00000010:1.0:1788971981.769348:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50f40e00.
00000400:00000200:1.0:1788971981.769358:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971981.769362:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971981.769369:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971981.769376:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179073024, portal 25
00000400:00000010:1.0:1788971981.769381:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588220.
00000100:00000200:1.0:1788971981.769384:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179073024, offset 0
00000400:00000200:1.0:1788971981.769393:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971981.769402:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971981.769409:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971981.769414:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971981.769418:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971981.769422:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971981.769425:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971981.769429:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [357:357:256:4294967295] 192.168.203.124@tcp LPNI seq info [357:357:8:4294967295]
00000400:00000200:1.0:1788971981.769433:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971981.769443:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971981.769450:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971981.769455:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400.
00000800:00000200:1.0:1788971981.769462:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971981.769469:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971981.769474:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971981.769496:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971981.769501:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9120.
00000100:00000010:1.0:1788971981.769586:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9540.
02000000:00000010:1.0:1788971981.769608:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50f41800.
00000400:00000200:1.0:1788971981.769612:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971981.769615:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78.
00000400:00000010:1.0:1788971981.769625:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588000.
00000100:00000200:1.0:1788971981.769628:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179073152, portal 10
00000400:00000010:1.0:1788971981.769634:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588cc0.
00000100:00000200:1.0:1788971981.769636:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179073152, offset 0
00000400:00000200:1.0:1788971981.769641:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971981.769645:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971981.769648:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971981.769651:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971981.769653:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971981.769655:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971981.769658:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971981.769660:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [358:358:255:4294967295] 192.168.203.124@tcp LPNI seq info [358:358:7:4294967295]
00000400:00000200:1.0:1788971981.769664:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971981.769670:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971981.769675:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971981.769678:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6c00.
00000800:00000200:1.0:1788971981.769682:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971981.769686:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971981.769689:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6c00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971981.769695:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:3.0F:1788971981.772098:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971981.772112:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400.
00000400:00000200:3.0:1788971981.772119:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971981.772127:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971981.772135:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588220
00000400:00000010:3.0:1788971981.772138:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588220.
00000100:00000200:3.0:1788971981.772149:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb7800 x1875872179073024/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971981.772291:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971981.772294:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6c00.
00000400:00000200:3.0:1788971981.772297:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971981.772322:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971981.772325:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588cc0
00000400:00000010:3.0:1788971981.772327:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588cc0.
00000100:00000200:3.0:1788971981.772334:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb6d80 x1875872179073152/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971997 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971981.774349:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.774358:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971981.774365:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.774369:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.774374:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.774378:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0b000
00000400:00000200:3.0:1788971981.774806:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.774813:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1345 [1] + 224
00000400:00000200:3.0:1788971981.774828:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971981.774832:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971981.774845:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb7800 x1875872179073024/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971997 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971981.776813:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971981.776839:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:1.0:1788971981.776841:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971981.776854:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb7800 x1875872179073024/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971997 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971981.776865:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb7800 x1875872179073024/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971997 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971981.776892:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50f40e00.
02000000:00000010:1.0:1788971981.776898:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51637a00.
00000100:00000010:1.0:1788971981.776907:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800.
00000400:00000200:3.0:1788971981.778054:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.778062:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:3.0:1788971981.778069:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.778083:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.778088:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.778147:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0b080
00000400:00000200:3.0:1788971981.778153:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971981.778159:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x134d [1] + 224
00000400:00000200:3.0:1788971981.778173:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971981.778178:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:3.0:1788971981.778189:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb6d80 x1875872179073152/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971997 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971981.781630:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78.
00000400:00000200:1.0:1788971981.781639:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588000
00000400:00000010:1.0:1788971981.781642:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588000.
00000100:00000200:1.0:1788971981.781655:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb6d80 x1875872179073152/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971997 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971981.781667:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb6d80 x1875872179073152/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971997 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971981.781700:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50f41800.
02000000:00000010:1.0:1788971981.781706:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51637100.
00000100:00000010:1.0:1788971981.781713:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80.
00000100:00000010:3.0:1788971982.471526:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9540.
00000100:00000010:2.0F:1788971982.476931:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9120.
00000100:00000010:1.0:1788971986.887586:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80.
02000000:00000010:1.0:1788971986.887612:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6400.
00000100:00000010:1.0:1788971986.887638:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800.
02000000:00000010:1.0:1788971986.887644:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6c00.
02000000:00000010:1.0:1788971986.887673:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50fa6800.
00000400:00000200:1.0:1788971986.887694:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971986.887707:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78.
00000400:00000010:1.0:1788971986.887715:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588000.
00000100:00000200:1.0:1788971986.887722:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179073280, portal 25
00000400:00000010:1.0:1788971986.887727:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770.
00000100:00000200:1.0:1788971986.887730:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179073280, offset 0
00000400:00000200:1.0:1788971986.887739:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971986.887747:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971986.887755:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971986.887760:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971986.887764:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971986.887767:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971986.887770:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971986.887773:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [359:359:256:4294967295] 192.168.203.124@tcp LPNI seq info [359:359:8:4294967295]
00000400:00000200:1.0:1788971986.887777:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971986.887786:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971986.887803:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971986.887809:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42b5e600.
00000800:00000200:1.0:1788971986.887816:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971986.887823:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971986.887827:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42b5e600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971986.887853:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:1.0:1788971986.887859:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d95a0.
00000100:00000010:1.0:1788971986.887868:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9720.
02000000:00000010:1.0:1788971986.887890:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50fa7800.
00000400:00000200:1.0:1788971986.887894:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:1.0:1788971986.887896:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688.
00000400:00000010:1.0:1788971986.887899:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588440.
00000100:00000200:1.0:1788971986.887902:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179073408, portal 10
00000400:00000010:1.0:1788971986.887907:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588908.
00000100:00000200:1.0:1788971986.887909:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179073408, offset 0
00000400:00000200:1.0:1788971986.887912:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:1.0:1788971986.887916:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971986.887919:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:1.0:1788971986.887921:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971986.887923:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:1.0:1788971986.887925:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971986.887927:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:1.0:1788971986.887930:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [360:360:255:4294967295] 192.168.203.124@tcp LPNI seq info [360:360:7:4294967295]
00000400:00000200:1.0:1788971986.887933:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:1.0:1788971986.887938:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971986.887943:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:1.0:1788971986.887946:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33ff00.
00000800:00000200:1.0:1788971986.887950:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:1.0:1788971986.887954:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:1.0:1788971986.887956:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33ff00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971986.887967:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000800:00000200:2.0:1788971986.888112:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971986.888117:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42b5e600.
00000400:00000200:2.0:1788971986.888122:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971986.888127:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971986.888133:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770
00000400:00000010:2.0:1788971986.888135:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770.
00000100:00000200:2.0:1788971986.888144:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb6d80 x1875872179073280/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971986.888220:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971986.888223:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33ff00.
00000400:00000200:2.0:1788971986.888227:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971986.888230:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971986.888233:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588908
00000400:00000010:2.0:1788971986.888235:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588908.
00000100:00000200:2.0:1788971986.888240:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb42fb7800 x1875872179073408/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972002 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971986.893626:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893636:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971986.893645:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893650:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893656:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893660:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0b100
00000400:00000200:2.0:1788971986.893665:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893671:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1355 [1] + 224
00000400:00000200:2.0:1788971986.893763:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971986.893769:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971986.893783:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb6d80 x1875872179073280/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972002 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971986.893819:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893822:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971986.893826:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893837:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893840:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893843:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0b180
00000400:00000200:2.0:1788971986.893847:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971986.893851:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x135d [1] + 224
00000400:00000200:2.0:1788971986.893859:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971986.893863:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971986.893869:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb42fb7800 x1875872179073408/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972002 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971986.893955:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688.
00000400:00000200:1.0:1788971986.893960:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588440
00000400:00000010:1.0:1788971986.893962:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588440.
00000100:00000200:1.0:1788971986.893969:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb7800 x1875872179073408/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972002 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971986.893979:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb7800 x1875872179073408/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972002 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971986.894008:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78.
00000400:00000200:1.0:1788971986.894011:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588000
00000400:00000010:1.0:1788971986.894013:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588000.
00000100:00000200:1.0:1788971986.894018:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb42fb6d80 x1875872179073280/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972002 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971986.894026:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb42fb6d80 x1875872179073280/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972002 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971986.894041:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50fa7800.
02000000:00000010:1.0:1788971986.894048:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6c00.
00000100:00000010:1.0:1788971986.894054:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800.
02000000:00000010:1.0:1788971986.894058:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50fa6800.
02000000:00000010:1.0:1788971986.894061:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6400.
00000100:00000010:1.0:1788971986.894063:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80.
00000100:00000010:2.0:1788971987.591435:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d95a0.
00000100:00000010:3.0:1788971987.591546:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9720.
00000001:02000400:0.0F:1788971991.551762:0:111969:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 301: Check for dynamic adds of same/wrong interface (memory leak) ========================================================== 12:39:50 (1788971990)
00000001:02000400:1.0:1788971991.793141:0:111983:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000100:00000010:2.0:1788971992.009146:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380.
02000000:00000010:2.0:1788971992.009166:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381dd00.
00000100:00000010:2.0:1788971992.009192:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412300.
02000000:00000010:2.0:1788971992.009209:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42b5e600.
02000000:00000010:2.0:1788971992.009242:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aac00.
00000400:00000200:2.0:1788971992.009252:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:2.0:1788971992.009257:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160.
00000400:00000010:2.0:1788971992.009264:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971992.009272:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179073536, portal 25
00000400:00000010:2.0:1788971992.009279:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18.
00000100:00000200:2.0:1788971992.009281:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179073536, offset 0
00000400:00000200:2.0:1788971992.009292:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
02000000:00000010:3.0:1788971992.009372:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017400.
00000400:00000200:3.0:1788971992.009378:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000010:3.0:1788971992.009382:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78f78.
00000400:00000010:3.0:1788971992.009386:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480815d8.
00000100:00000200:3.0:1788971992.009389:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179073664, portal 10
00000400:00000010:3.0:1788971992.009395:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081220.
00000100:00000200:3.0:1788971992.009398:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179073664, offset 0
00000400:00000200:3.0:1788971992.009404:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp
00000400:00000200:2.0:1788971992.009494:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.009503:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:2.0:1788971992.009515:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971992.009519:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:2.0:1788971992.009527:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971992.009530:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:2.0:1788971992.009534:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [361:361:256:4294967295] 192.168.203.124@tcp LPNI seq info [361:361:8:4294967295]
00000400:00000200:2.0:1788971992.009538:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.009547:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971992.009561:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:2.0:1788971992.009568:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53519e00.
00000800:00000200:2.0:1788971992.009575:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:2.0:1788971992.009583:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:2.0:1788971992.009587:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53519e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971992.009601:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000100:00000010:2.0:1788971992.009607:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7ea0.
00000100:00000010:2.0:1788971992.009616:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840.
00000800:00000200:2.0:1788971992.009739:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971992.009743:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53519e00.
00000400:00000200:2.0:1788971992.009748:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971992.009753:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:3.0:1788971992.010838:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971992.010845:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.124@tcp local destination
00000400:00000200:3.0:1788971992.010849:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971992.010852:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp
00000400:00000200:3.0:1788971992.010855:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971992.010857:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp
00000400:00000200:3.0:1788971992.010860:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [362:362:255:4294967295] 192.168.203.124@tcp LPNI seq info [362:362:7:4294967295]
00000400:00000200:3.0:1788971992.010864:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:3.0:1788971992.010870:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:<?>) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971992.010875:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp
00000800:00000010:3.0:1788971992.010882:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500.
00000800:00000200:3.0:1788971992.010896:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5)
00000800:00000200:3.0:1788971992.010902:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988
00000800:00000200:3.0:1788971992.010905:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971992.010912:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.011088:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18
00000400:00000010:2.0:1788971992.011092:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18.
00000100:00000200:2.0:1788971992.011103:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43410380 x1875872179073536/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971992.013108:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971992.013117:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500.
00000400:00000200:2.0:1788971992.013122:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971992.013129:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000400:00000200:2.0:1788971992.013135:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081220
00000400:00000010:2.0:1788971992.013137:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081220.
00000100:00000200:2.0:1788971992.013147:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff98bb43412300 x1875872179073664/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972008 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971992.015625:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015633:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971992.015641:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015646:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015651:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015654:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0b200
00000400:00000200:2.0:1788971992.015659:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015682:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1365 [1] + 224
00000400:00000200:2.0:1788971992.015695:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971992.015699:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971992.015711:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43410380 x1875872179073536/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972008 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971992.015738:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015742:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me
00000400:00000200:2.0:1788971992.015746:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015748:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015751:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015753:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0b280
00000400:00000200:2.0:1788971992.015756:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1
00000400:00000200:2.0:1788971992.015760:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1375 [1] + 224
00000400:00000200:2.0:1788971992.015769:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971992.015772:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK
00000100:00000200:2.0:1788971992.015778:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff98bb43412300 x1875872179073664/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972008 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971992.015810:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78f78.
00000400:00000200:2.0:1788971992.015814:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480815d8
00000400:00000010:2.0:1788971992.015817:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480815d8.
00000100:00000200:2.0:1788971992.015822:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43412300 x1875872179073664/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972008 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971992.015830:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43412300 x1875872179073664/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972008 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971992.015861:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017400.
02000000:00000010:2.0:1788971992.015867:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42b5e600.
00000100:00000010:2.0:1788971992.015872:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412300.
00000400:00000010:2.0:1788971992.015889:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160.
00000400:00000200:2.0:1788971992.015892:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0
00000400:00000010:2.0:1788971992.015900:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0.
00000100:00000200:2.0:1788971992.015906:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff98bb43410380 x1875872179073536/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972008 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971992.015914:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff98bb43410380 x1875872179073536/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972008 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971992.015925:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aac00.
02000000:00000010:2.0:1788971992.015928:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381dd00.
00000100:00000010:2.0:1788971992.015932:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380.
00000100:00000010:3.0:1788971992.712040:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840.
00000100:00000010:2.0:1788971992.712575:0:109572:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7ea0.
00000001:02000400:3.0:1788971993.109021:0:112109:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_301: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 321 lines, 321 kept, 0 dropped, 0 bad.