00000100:00000010:2.0F:1788971547.922424:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180.
02000000:00000010:2.0:1788971547.922453:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63000.
00000100:00000010:2.0:1788971547.922493:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680.
02000000:00000010:2.0:1788971547.922500:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63800.
00000100:00000010:2.0:1788971547.922509:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ea00.
02000000:00000010:2.0:1788971547.922513:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63b00.
02000000:00000010:2.0:1788971547.922544:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600.
00000400:00000200:2.0:1788971547.922553:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971547.922557:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60.
00000400:00000010:2.0:1788971547.922564:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110.
00000100:00000200:2.0:1788971547.922571:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189543936, portal 10
00000400:00000010:2.0:1788971547.922576:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
02000000:00000010:3.0F:1788971547.922578:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002c00.
00000100:00000200:2.0:1788971547.922579:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189543936, offset 0
00000400:00000200:2.0:1788971547.922587:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971547.922589:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971547.922591:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160.
00000400:00000200:2.0:1788971547.922594:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971547.922595:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788971547.922598:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189543808, portal 25
00000400:00000010:3.0:1788971547.922602:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0.
00000400:00000200:2.0:1788971547.922602:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000100:00000200:3.0:1788971547.922605:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189543808, offset 0
00000400:00000200:2.0:1788971547.922607:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971547.922609:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971547.922611:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971547.922615:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971547.922617:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971547.922621:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [240:240:256:4294967295] 192.168.203.150@tcp LPNI seq info [240:240:8:4294967295]
00000400:00000200:2.0:1788971547.922625:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971547.922634:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000400:00000200:3.0:1788971547.922638:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971547.922639:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971547.922641:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000800:00000010:2.0:1788971547.922643:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63a00.
00000400:00000200:3.0:1788971547.922644:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971547.922646:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971547.922648:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971547.922650:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000800:00000200:2.0:1788971547.922650:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000400:00000200:3.0:1788971547.922652:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [241:241:255:4294967295] 192.168.203.150@tcp LPNI seq info [241:241:7:4294967295]
00000400:00000200:3.0:1788971547.922655:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971547.922657:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:3.0:1788971547.922661:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971547.922662:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63a00 type 1, nob 320 niov 1 nkiov 1
00000800:00000200:3.0:1788971547.922665:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971547.922668:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100.
00000800:00000200:3.0:1788971547.922671:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971547.922674:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:2.0:1788971547.922675:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971547.922677:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971547.922680:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971547.922680:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726cc00.
00000100:00000010:2.0:1788971547.922689:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726cd80.
02000000:00000010:3.0:1788971547.922700:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002800.
00000400:00000200:3.0:1788971547.922703:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971547.922705:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688.
00000400:00000010:3.0:1788971547.922708:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788971547.922710:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544064, portal 10
00000400:00000010:3.0:1788971547.922715:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48736e8.
00000100:00000200:3.0:1788971547.922717:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189544064, offset 0
00000400:00000200:3.0:1788971547.922721:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971547.922724:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.922733:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971547.922736:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971547.922738:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000800:00000200:2.0:1788971547.922738:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000400:00000200:3.0:1788971547.922741:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971547.922743:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000800:00000010:2.0:1788971547.922745:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63a00.
00000400:00000200:3.0:1788971547.922747:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [242:242:254:4294967295] 192.168.203.150@tcp LPNI seq info [242:242:6:4294967295]
00000400:00000200:2.0:1788971547.922749:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971547.922750:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971547.922754:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971547.922756:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971547.922760:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971547.922763:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971547.922766:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000800:00000200:3.0:1788971547.922767:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000400:00000010:2.0:1788971547.922768:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000800:00000200:3.0:1788971547.922771:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971547.922774:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971547.922779:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000200:2.0:1788971547.922794:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840e680 x1875872189543936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971547.922834:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971547.922837:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100.
00000400:00000200:2.0:1788971547.922840:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971547.922844:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971547.922847:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0
00000400:00000010:2.0:1788971547.922848:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0.
00000100:00000200:2.0:1788971547.922853:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840d180 x1875872189543808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971547.922930:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971547.922933:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971547.922936:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971547.922939:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971547.922942:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48736e8
00000400:00000010:2.0:1788971547.922943:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48736e8.
00000100:00000200:2.0:1788971547.922948:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ea00 x1875872189544064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971547.925634:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925643:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971547.925650:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925654:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925660:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925664:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807580
00000400:00000200:3.0:1788971547.925711:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925717:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xc7d [1] + 224
00000400:00000200:3.0:1788971547.925729:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971547.925733:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971547.925748:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840d180 x1875872189543808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971547.925777:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925781:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971547.925785:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925794:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925798:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925800:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807600
00000400:00000200:3.0:1788971547.925804:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925808:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xc6d [1] + 224
00000400:00000200:3.0:1788971547.925837:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971547.925842:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971547.925848:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840e680 x1875872189543936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971547.925882:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925887:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971547.925891:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925894:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925897:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925900:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807680
00000400:00000200:3.0:1788971547.925903:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971547.925906:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xc8d [1] + 224
00000400:00000200:3.0:1788971547.925914:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971547.925917:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971547.925923:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ea00 x1875872189544064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971547.925955:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60.
00000400:00000200:3.0:1788971547.925960:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110
00000400:00000010:3.0:1788971547.925963:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110.
00000100:00000200:3.0:1788971547.925969:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840e680 x1875872189543936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971547.925978:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840e680 x1875872189543936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971547.926011:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600.
02000000:00000010:3.0:1788971547.926022:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63800.
00000100:00000010:3.0:1788971547.926026:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680.
00000400:00000010:3.0:1788971547.926046:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160.
00000400:00000200:3.0:1788971547.926049:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8
00000400:00000010:3.0:1788971547.926051:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788971547.926058:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840d180 x1875872189543808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971547.926066:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840d180 x1875872189543808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971547.926078:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688.
00000400:00000200:3.0:1788971547.926080:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8
00000400:00000010:3.0:1788971547.926081:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788971547.926086:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ea00 x1875872189544064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971547.926093:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ea00 x1875872189544064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971563 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971547.926105:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002c00.
02000000:00000010:3.0:1788971547.926108:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63000.
00000100:00000010:3.0:1788971547.926110:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180.
02000000:00000010:3.0:1788971547.926113:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002800.
02000000:00000010:3.0:1788971547.926116:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63b00.
00000100:00000010:3.0:1788971547.926118:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ea00.
00000001:02000400:3.0:1788971548.026916:0:109690:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 280: Don't panic when request_module fails ========================================================== 12:32:26 (1788971546)
00000020:00000010:3.0:1788971548.254238:0:109715:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971548.254337:0:109715:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000020:00000010:2.0:1788971548.320705:0:109721:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971548.320797:0:109721:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000100:00000010:3.0:1788971548.945987:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726cd80.
00000100:00000010:2.0:1788971548.949198:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726cc00.
00000020:00000010:1.0F:1788971549.349384:0:109725:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971549.349471:0:109725:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:3.0:1788971550.398107:0:109730:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971550.398197:0:109730:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000020:00000010:1.0:1788971551.440786:0:109734:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971551.441738:0:109734:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:0.0F:1788971552.495269:0:109738:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780410.
00000020:00000010:0.0:1788971552.495370:0:109738:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780410.
00000100:00000010:0.0:1788971553.043309:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500.
02000000:00000010:0.0:1788971553.043331:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b000.
00000100:00000010:0.0:1788971553.043378:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700.
02000000:00000010:0.0:1788971553.043383:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bd00.
00000100:00000010:0.0:1788971553.043389:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800.
02000000:00000010:0.0:1788971553.043393:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100.
02000000:00000010:0.0:1788971553.043425:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4000.
00000400:00000200:0.0:1788971553.043433:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971553.043437:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971553.043443:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8.
00000100:00000200:0.0:1788971553.043450:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544320, portal 10
00000400:00000010:0.0:1788971553.043457:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8198.
00000100:00000200:0.0:1788971553.043460:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189544320, offset 0
00000400:00000200:0.0:1788971553.043468:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971553.043477:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971553.043485:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971553.043490:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971553.043494:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971553.043497:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971553.043500:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971553.043504:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [243:243:256:4294967295] 192.168.203.150@tcp LPNI seq info [243:243:8:4294967295]
00000400:00000200:0.0:1788971553.043509:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971553.043518:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971553.043524:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971553.043531:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ec00.
00000800:00000200:0.0:1788971553.043537:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971553.043543:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971553.043547:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ec00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971553.043576:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971553.043582:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c51e0.
00000100:00000010:0.0:1788971553.043607:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c56c0.
02000000:00000010:0.0:1788971553.043637:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5c00.
00000400:00000200:0.0:1788971553.043642:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971553.043645:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971553.043649:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38.
00000100:00000200:0.0:1788971553.043652:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544192, portal 25
00000400:00000010:0.0:1788971553.043655:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8770.
00000100:00000200:0.0:1788971553.043657:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189544192, offset 0
00000400:00000200:0.0:1788971553.043662:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971553.043666:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971553.043669:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971553.043672:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971553.043674:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971553.043677:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971553.043679:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971553.043681:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [244:244:255:4294967295] 192.168.203.150@tcp LPNI seq info [244:244:7:4294967295]
00000400:00000200:0.0:1788971553.043685:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971553.043691:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971553.043696:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971553.043699:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1e900.
00000800:00000200:0.0:1788971553.043703:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971553.043708:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971553.043711:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1e900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971553.043715:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971553.043726:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5a00.
00000400:00000200:0.0:1788971553.043729:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971553.043731:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971553.043734:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971553.043737:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544448, portal 10
00000400:00000010:0.0:1788971553.043741:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8440.
00000100:00000200:0.0:1788971553.043743:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189544448, offset 0
00000400:00000200:0.0:1788971553.043754:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971553.043757:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971553.043761:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971553.043763:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971553.043765:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971553.043767:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971553.043769:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971553.043771:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [245:245:254:4294967295] 192.168.203.150@tcp LPNI seq info [245:245:6:4294967295]
00000400:00000200:0.0:1788971553.043774:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971553.043780:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971553.043783:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971553.043786:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ee00.
00000800:00000200:0.0:1788971553.043789:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971553.043793:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971553.043795:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971553.043799:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971553.043996:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971553.044001:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ec00.
00000400:00000200:2.0:1788971553.044007:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971553.044012:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971553.044017:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8198
00000400:00000010:2.0:1788971553.044019:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8198.
00000100:00000200:2.0:1788971553.044033:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88700 x1875872189544320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971553.044058:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971553.044060:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1e900.
00000400:00000200:2.0:1788971553.044063:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971553.044067:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971553.044069:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8770
00000400:00000010:2.0:1788971553.044071:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8770.
00000100:00000200:2.0:1788971553.044074:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a89500 x1875872189544192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971553.044134:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971553.044137:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ee00.
00000400:00000200:2.0:1788971553.044140:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971553.044143:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971553.044146:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8440
00000400:00000010:2.0:1788971553.044147:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8440.
00000100:00000200:2.0:1788971553.044151:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a8b800 x1875872189544448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971553.049520:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971553.049541:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971553.049550:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971553.049555:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971553.049559:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971553.049563:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807780
00000400:00000200:3.0:1788971553.049611:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971553.049617:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xc9d [1] + 224
00000400:00000200:3.0:1788971553.049631:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971553.049636:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971553.049649:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88700 x1875872189544320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971553.049860:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971553.049866:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8
00000400:00000010:0.0:1788971553.049868:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8.
00000100:00000200:0.0:1788971553.049875:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88700 x1875872189544320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971553.049885:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88700 x1875872189544320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971553.049915:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4000.
02000000:00000010:0.0:1788971553.049921:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bd00.
00000100:00000010:0.0:1788971553.049926:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700.
00000400:00000200:2.0:1788971553.050581:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050587:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971553.050592:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050595:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050599:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050602:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807700
00000400:00000200:2.0:1788971553.050607:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050612:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xca5 [1] + 224
00000400:00000200:2.0:1788971553.050621:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971553.050625:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971553.050643:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a89500 x1875872189544192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971553.050693:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050697:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971553.050701:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050703:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050705:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050708:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807800
00000400:00000200:2.0:1788971553.050712:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971553.050715:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xcad [1] + 224
00000400:00000200:2.0:1788971553.050723:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971553.050726:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971553.050731:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a8b800 x1875872189544448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971553.050800:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971553.050804:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38
00000400:00000010:0.0:1788971553.050806:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38.
00000100:00000200:0.0:1788971553.050845:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a89500 x1875872189544192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971553.050860:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a89500 x1875872189544192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971553.050878:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971553.050882:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58
00000400:00000010:0.0:1788971553.050883:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971553.050887:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a8b800 x1875872189544448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971553.050895:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a8b800 x1875872189544448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971569 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971553.050910:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5c00.
02000000:00000010:0.0:1788971553.050915:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b000.
00000100:00000010:0.0:1788971553.050918:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500.
02000000:00000010:0.0:1788971553.050922:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5a00.
02000000:00000010:0.0:1788971553.050924:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100.
00000100:00000010:0.0:1788971553.050925:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800.
00000020:00000010:2.0:1788971553.528030:0:109742:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971553.528119:0:109742:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000100:00000010:3.0:1788971554.065943:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c56c0.
00000100:00000010:2.0:1788971554.065943:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c51e0.
00000020:00000010:3.0:1788971554.572083:0:109746:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971554.572176:0:109746:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971555.605092:0:109750:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971555.605252:0:109750:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000020:00000010:2.0:1788971556.662723:0:109754:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971556.666014:0:109754:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000020:00000010:3.0:1788971557.713395:0:109759:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971557.714493:0:109759:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:1.0:1788971558.162533:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480.
02000000:00000010:1.0:1788971558.162555:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fb00.
00000100:00000010:1.0:1788971558.162663:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100.
02000000:00000010:1.0:1788971558.162670:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f000.
00000100:00000010:1.0:1788971558.162678:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788971558.162684:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f600.
02000000:00000010:1.0:1788971558.162787:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17ae200.
00000400:00000200:1.0:1788971558.162805:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971558.162809:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000010:1.0:1788971558.162854:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971558.162862:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544704, portal 10
00000400:00000010:1.0:1788971558.162869:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000.
00000100:00000200:1.0:1788971558.162873:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189544704, offset 0
00000400:00000200:1.0:1788971558.162882:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971558.162891:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971558.162897:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971558.162903:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971558.162907:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971558.162910:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971558.162913:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971558.162916:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [246:246:256:4294967295] 192.168.203.150@tcp LPNI seq info [246:246:8:4294967295]
00000400:00000200:1.0:1788971558.162921:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971558.162930:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971558.162936:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971558.162941:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f100.
00000800:00000200:1.0:1788971558.162947:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971558.162954:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971558.162958:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971558.162993:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:1.0:1788971558.162999:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467660.
00000100:00000010:1.0:1788971558.163018:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467e40.
02000000:00000010:1.0:1788971558.163041:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17ae000.
00000400:00000200:1.0:1788971558.163045:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971558.163047:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000010:1.0:1788971558.163051:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880.
00000100:00000200:1.0:1788971558.163054:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544576, portal 25
00000400:00000010:1.0:1788971558.163057:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68.
00000100:00000200:1.0:1788971558.163059:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189544576, offset 0
00000400:00000200:1.0:1788971558.163064:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971558.163067:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971558.163071:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971558.163074:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971558.163076:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971558.163079:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971558.163080:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971558.163083:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [247:247:255:4294967295] 192.168.203.150@tcp LPNI seq info [247:247:7:4294967295]
00000400:00000200:1.0:1788971558.163086:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971558.163092:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971558.163096:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971558.163099:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f400.
00000800:00000200:1.0:1788971558.163103:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971558.163107:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971558.163110:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971558.163114:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:1.0:1788971558.163124:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17af600.
00000400:00000200:1.0:1788971558.163128:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971558.163130:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738.
00000400:00000010:1.0:1788971558.163134:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0.
00000100:00000200:1.0:1788971558.163136:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544832, portal 10
00000400:00000010:1.0:1788971558.163146:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8.
00000100:00000200:1.0:1788971558.163149:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189544832, offset 0
00000400:00000200:1.0:1788971558.163153:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971558.163157:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971558.163161:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971558.163163:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971558.163166:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971558.163168:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971558.163170:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971558.163173:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [248:248:254:4294967295] 192.168.203.150@tcp LPNI seq info [248:248:6:4294967295]
00000400:00000200:1.0:1788971558.163176:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971558.163183:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971558.163187:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971558.163190:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f200.
00000800:00000200:1.0:1788971558.163193:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971558.163197:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971558.163200:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971558.163204:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971558.163401:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971558.163406:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f100.
00000400:00000200:3.0:1788971558.163413:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971558.163418:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971558.163423:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000
00000400:00000010:3.0:1788971558.163425:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000.
00000100:00000200:3.0:1788971558.163434:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3100 x1875872189544704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971558.163454:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971558.163457:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f400.
00000400:00000200:3.0:1788971558.163459:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971558.163462:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971558.163464:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68
00000400:00000010:3.0:1788971558.163472:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68.
00000100:00000200:3.0:1788971558.163478:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3480 x1875872189544576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971558.163546:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971558.163549:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f200.
00000400:00000200:3.0:1788971558.163553:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971558.163556:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971558.163559:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8
00000400:00000010:3.0:1788971558.163560:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8.
00000100:00000200:3.0:1788971558.163565:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189544832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971558.165648:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971558.165658:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971558.165667:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971558.165672:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971558.165677:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971558.165682:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807880
00000400:00000200:2.0:1788971558.165687:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971558.165692:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xcbd [1] + 224
00000400:00000200:2.0:1788971558.165707:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971558.165724:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971558.165735:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3480 x1875872189544576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971558.166018:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000200:1.0:1788971558.166024:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880
00000400:00000010:1.0:1788971558.166026:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880.
00000100:00000200:1.0:1788971558.166032:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3480 x1875872189544576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971558.166041:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3480 x1875872189544576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971558.166067:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17ae000.
02000000:00000010:1.0:1788971558.166072:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fb00.
00000100:00000010:1.0:1788971558.166076:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480.
00000400:00000200:3.0:1788971558.171042:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171052:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971558.171060:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171064:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171069:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171073:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807900
00000400:00000200:3.0:1788971558.171077:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171089:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xcb5 [1] + 224
00000400:00000200:3.0:1788971558.171099:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971558.171106:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971558.171119:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3100 x1875872189544704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971558.171156:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171159:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971558.171178:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171180:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171183:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171186:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807980
00000400:00000200:3.0:1788971558.171189:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971558.171207:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xcc5 [1] + 224
00000400:00000200:3.0:1788971558.171218:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971558.171221:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971558.171228:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189544832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971558.171356:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738.
00000400:00000200:1.0:1788971558.171362:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0
00000400:00000010:1.0:1788971558.171371:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0.
00000100:00000200:1.0:1788971558.171378:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189544832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971558.171388:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189544832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971558.171415:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000200:1.0:1788971558.171417:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220
00000400:00000010:1.0:1788971558.171419:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971558.171423:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3100 x1875872189544704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971558.171432:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3100 x1875872189544704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971574 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971558.171447:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17af600.
02000000:00000010:1.0:1788971558.171453:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f600.
00000100:00000010:1.0:1788971558.171457:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788971558.171461:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17ae200.
02000000:00000010:1.0:1788971558.171463:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f000.
00000100:00000010:1.0:1788971558.171465:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100.
00000020:00000010:3.0:1788971558.774300:0:109763:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971558.774396:0:109763:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:2.0:1788971559.185942:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467660.
00000100:00000010:3.0:1788971559.185943:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467e40.
00000020:00000010:3.0:1788971559.800051:0:109767:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971559.800156:0:109767:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000020:00000010:1.0:1788971560.831284:0:109771:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971560.831384:0:109771:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971561.866852:0:109775:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971561.866957:0:109775:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:0.0:1788971562.898249:0:109779:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780410.
00000020:00000010:0.0:1788971562.898343:0:109779:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780410.
00000100:00000010:1.0:1788971563.282107:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100.
02000000:00000010:1.0:1788971563.282130:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f000.
00000100:00000010:1.0:1788971563.282176:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788971563.282181:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f600.
00000100:00000010:1.0:1788971563.282188:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480.
02000000:00000010:1.0:1788971563.282192:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fb00.
02000000:00000010:0.0:1788971563.283052:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4000.
00000400:00000200:0.0:1788971563.283064:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971563.283067:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971563.283073:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971563.283080:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189544960, portal 25
00000400:00000010:0.0:1788971563.283086:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38.
00000100:00000200:0.0:1788971563.283090:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189544960, offset 0
00000400:00000200:0.0:1788971563.283098:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971563.283113:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971563.283121:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971563.283126:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971563.283130:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971563.283148:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971563.283151:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971563.283154:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [249:249:256:4294967295] 192.168.203.150@tcp LPNI seq info [249:249:8:4294967295]
00000400:00000200:0.0:1788971563.283158:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971563.283168:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971563.283173:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971563.283178:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ee00.
00000800:00000200:0.0:1788971563.283184:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971563.283191:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971563.283195:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971563.283268:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971563.283274:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d20.
00000100:00000010:0.0:1788971563.283285:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5ae0.
02000000:00000010:0.0:1788971563.283296:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4c00.
00000400:00000200:0.0:1788971563.283300:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971563.283303:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971563.283306:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8.
00000100:00000200:0.0:1788971563.283309:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545216, portal 10
00000400:00000010:0.0:1788971563.283314:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8.
00000100:00000200:0.0:1788971563.283316:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189545216, offset 0
00000400:00000200:0.0:1788971563.283329:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971563.283333:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971563.283337:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971563.283340:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971563.283342:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971563.283345:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971563.283347:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971563.283350:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [250:250:255:4294967295] 192.168.203.150@tcp LPNI seq info [250:250:7:4294967295]
00000400:00000200:0.0:1788971563.283354:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971563.283360:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971563.283373:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971563.283376:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1e900.
00000800:00000200:0.0:1788971563.283380:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971563.283384:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971563.283387:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1e900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971563.283393:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971563.283412:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5000.
00000400:00000200:0.0:1788971563.283416:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971563.283418:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971563.283421:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8dd0.
00000100:00000200:0.0:1788971563.283424:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545088, portal 10
00000400:00000010:0.0:1788971563.283428:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8ee0.
00000100:00000200:0.0:1788971563.283431:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189545088, offset 0
00000400:00000200:0.0:1788971563.283435:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971563.283438:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971563.283442:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971563.283445:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971563.283447:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971563.283450:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971563.283452:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971563.283455:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [251:251:254:4294967295] 192.168.203.150@tcp LPNI seq info [251:251:6:4294967295]
00000400:00000200:0.0:1788971563.283459:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971563.283465:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971563.283469:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971563.283472:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ec00.
00000800:00000200:0.0:1788971563.283476:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971563.283480:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971563.283482:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ec00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971563.283487:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971563.284075:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971563.284082:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ee00.
00000400:00000200:2.0:1788971563.284088:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971563.284094:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971563.284110:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38
00000400:00000010:2.0:1788971563.284114:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38.
00000100:00000200:2.0:1788971563.284124:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3100 x1875872189544960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971563.284149:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971563.284152:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1e900.
00000400:00000200:2.0:1788971563.284155:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971563.284158:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971563.284161:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8
00000400:00000010:2.0:1788971563.284162:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8.
00000100:00000200:2.0:1788971563.284166:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3480 x1875872189545216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971563.284535:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971563.284539:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ec00.
00000400:00000200:2.0:1788971563.284543:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971563.284547:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971563.284615:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8ee0
00000400:00000010:2.0:1788971563.284618:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8ee0.
00000100:00000200:2.0:1788971563.284624:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189545088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971563.291546:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291566:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971563.291576:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291581:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291586:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291590:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807b00
00000400:00000200:3.0:1788971563.291595:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291602:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xcd5 [1] + 224
00000400:00000200:3.0:1788971563.291614:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971563.291618:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971563.291633:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3480 x1875872189545216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971563.291670:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291674:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971563.291678:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291680:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291683:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291686:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807a80
00000400:00000200:3.0:1788971563.291690:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971563.291693:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xcdd [1] + 224
00000400:00000200:3.0:1788971563.291718:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971563.291722:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971563.291729:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189545088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971563.291942:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971563.291965:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8dd0
00000400:00000010:0.0:1788971563.291968:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8dd0.
00000100:00000200:0.0:1788971563.291975:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189545088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971563.291984:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189545088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971563.292017:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971563.292020:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8
00000400:00000010:0.0:1788971563.292022:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8.
00000100:00000200:0.0:1788971563.292026:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3480 x1875872189545216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971563.292034:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3480 x1875872189545216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971563.292060:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5000.
02000000:00000010:0.0:1788971563.292067:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f600.
00000100:00000010:0.0:1788971563.292072:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
02000000:00000010:0.0:1788971563.292075:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4c00.
02000000:00000010:0.0:1788971563.292077:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fb00.
00000100:00000010:0.0:1788971563.292079:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480.
00000400:00000200:2.0:1788971563.304830:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971563.304840:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971563.304850:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971563.304855:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971563.304860:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971563.304875:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807a00
00000400:00000200:2.0:1788971563.304881:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971563.304886:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xccd [1] + 224
00000400:00000200:2.0:1788971563.304900:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971563.304905:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971563.304917:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3100 x1875872189544960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971563.305067:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971563.305073:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58
00000400:00000010:0.0:1788971563.305075:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971563.305082:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3100 x1875872189544960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971563.305092:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3100 x1875872189544960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971579 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971563.305179:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4000.
02000000:00000010:0.0:1788971563.305187:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f000.
00000100:00000010:0.0:1788971563.305192:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100.
00000020:00000010:1.0:1788971563.925244:0:109783:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971563.925326:0:109783:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000100:00000010:3.0:1788971564.305942:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5ae0.
00000100:00000010:2.0:1788971564.305942:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d20.
00000020:00000010:1.0:1788971564.949166:0:109788:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971564.949258:0:109788:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971565.984041:0:109792:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971565.984213:0:109792:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:2.0:1788971567.037479:0:109796:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971567.037579:0:109796:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971568.071402:0:109800:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971568.074920:0:109800:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000100:00000010:1.0:1788971568.402264:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00.
02000000:00000010:1.0:1788971568.402289:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f800.
00000100:00000010:1.0:1788971568.402323:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100.
02000000:00000010:1.0:1788971568.402329:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00.
00000100:00000010:1.0:1788971568.402335:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480.
02000000:00000010:1.0:1788971568.402339:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f900.
02000000:00000010:0.0:1788971568.402447:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4a00.
00000400:00000200:0.0:1788971568.402473:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971568.402478:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971568.402484:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971568.402491:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545344, portal 25
00000400:00000010:0.0:1788971568.402497:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8.
00000100:00000200:0.0:1788971568.402501:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189545344, offset 0
00000400:00000200:0.0:1788971568.402509:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971568.402518:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971568.402526:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971568.402531:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971568.402535:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971568.402539:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971568.402542:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971568.402546:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [252:252:256:4294967295] 192.168.203.150@tcp LPNI seq info [252:252:8:4294967295]
00000400:00000200:0.0:1788971568.402550:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971568.402559:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971568.402566:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971568.402571:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ec00.
00000800:00000200:0.0:1788971568.402577:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971568.402584:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971568.402588:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ec00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971568.402610:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971568.402615:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5ae0.
00000100:00000010:0.0:1788971568.402637:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d20.
02000000:00000010:0.0:1788971568.402649:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5000.
00000400:00000200:0.0:1788971568.402654:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971568.402656:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971568.402660:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8dd0.
00000100:00000200:0.0:1788971568.402663:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545600, portal 10
00000400:00000010:0.0:1788971568.402668:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8908.
00000100:00000200:0.0:1788971568.402671:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189545600, offset 0
00000400:00000200:0.0:1788971568.402682:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971568.402686:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971568.402690:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971568.402692:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971568.402695:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971568.402697:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971568.402699:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971568.402702:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [253:253:255:4294967295] 192.168.203.150@tcp LPNI seq info [253:253:7:4294967295]
00000400:00000200:0.0:1788971568.402705:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971568.402712:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971568.402730:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971568.402736:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1e900.
00000800:00000200:0.0:1788971568.402740:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971568.402745:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971568.402747:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1e900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971568.402753:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971568.402773:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5600.
00000400:00000200:0.0:1788971568.402776:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971568.402779:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971568.402782:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971568.402785:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545472, portal 10
00000400:00000010:0.0:1788971568.402789:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8660.
00000100:00000200:0.0:1788971568.402791:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189545472, offset 0
00000400:00000200:0.0:1788971568.402795:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971568.402798:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971568.402801:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971568.402805:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971568.402808:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971568.402809:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971568.402812:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971568.403799:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [254:254:254:4294967295] 192.168.203.150@tcp LPNI seq info [254:254:6:4294967295]
00000400:00000200:0.0:1788971568.403835:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971568.403846:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971568.403851:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971568.403857:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ee00.
00000800:00000200:0.0:1788971568.403862:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971568.403868:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971568.403871:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971568.403878:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971568.407373:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971568.407384:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ec00.
00000400:00000200:3.0:1788971568.407394:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971568.407403:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971568.407410:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8
00000400:00000010:3.0:1788971568.407413:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8.
00000100:00000200:3.0:1788971568.407426:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f1c00 x1875872189545344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971568.407456:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971568.407459:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1e900.
00000400:00000200:3.0:1788971568.407462:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971568.407466:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971568.407469:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8908
00000400:00000010:3.0:1788971568.407470:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8908.
00000100:00000200:3.0:1788971568.407475:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3480 x1875872189545600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971568.407550:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971568.407553:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ee00.
00000400:00000200:3.0:1788971568.407556:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971568.407559:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971568.407562:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8660
00000400:00000010:3.0:1788971568.407564:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8660.
00000100:00000200:3.0:1788971568.407569:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3100 x1875872189545472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971568.411218:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411227:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971568.411235:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411240:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411246:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411250:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807c80
00000400:00000200:2.0:1788971568.411254:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411261:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xced [1] + 224
00000400:00000200:2.0:1788971568.411271:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971568.411286:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971568.411301:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3480 x1875872189545600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971568.411341:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411345:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971568.411349:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411352:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411355:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411357:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807b80
00000400:00000200:2.0:1788971568.411375:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971568.411379:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xce5 [1] + 224
00000400:00000200:2.0:1788971568.411390:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971568.411393:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971568.411401:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f1c00 x1875872189545344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971568.411489:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971568.411495:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58
00000400:00000010:0.0:1788971568.411498:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971568.411505:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f1c00 x1875872189545344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971568.411514:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f1c00 x1875872189545344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971568.411537:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971568.411540:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8dd0
00000400:00000010:0.0:1788971568.411541:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8dd0.
00000100:00000200:0.0:1788971568.411546:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3480 x1875872189545600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971568.411553:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3480 x1875872189545600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971568.411571:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4a00.
02000000:00000010:0.0:1788971568.411576:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f800.
00000100:00000010:0.0:1788971568.411580:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00.
02000000:00000010:0.0:1788971568.411584:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5000.
02000000:00000010:0.0:1788971568.411586:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f900.
00000100:00000010:0.0:1788971568.411587:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480.
00000400:00000200:3.0:1788971568.415680:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971568.415702:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971568.415710:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971568.415714:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971568.415720:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971568.415735:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807c00
00000400:00000200:3.0:1788971568.415740:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971568.415746:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xcf5 [1] + 224
00000400:00000200:3.0:1788971568.415761:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971568.415766:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971568.415780:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3100 x1875872189545472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971568.415920:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971568.415930:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971568.415932:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971568.415939:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3100 x1875872189545472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971568.415947:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3100 x1875872189545472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971584 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971568.415977:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5600.
02000000:00000010:0.0:1788971568.415983:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00.
00000100:00000010:0.0:1788971568.415988:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100.
00000020:00000010:0.0:1788971569.112373:0:109804:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780410.
00000020:00000010:0.0:1788971569.112470:0:109804:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780410.
00000100:00000010:2.0:1788971569.426329:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5ae0.
00000100:00000010:3.0:1788971569.426923:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d20.
00000020:00000010:0.0:1788971570.157673:0:109808:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780410.
00000020:00000010:0.0:1788971570.157789:0:109808:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780410.
00000020:00000010:2.0:1788971571.202052:0:109812:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971571.202163:0:109812:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971572.269774:0:109817:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971572.270415:0:109817:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000020:00000010:1.0:1788971573.294535:0:109821:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971573.294656:0:109821:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000100:00000010:3.0:1788971573.522751:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380.
02000000:00000010:3.0:1788971573.522783:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600.
02000000:00000010:2.0:1788971573.522895:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9ec00.
00000400:00000200:2.0:1788971573.522910:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971573.522914:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126a50.
00000400:00000010:2.0:1788971573.522919:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971573.522926:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545728, portal 25
00000400:00000010:2.0:1788971573.522931:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788971573.522934:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189545728, offset 0
00000400:00000200:2.0:1788971573.522942:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971573.522950:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.522970:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971573.522975:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971573.522981:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971573.522984:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971573.522987:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971573.522990:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [255:255:256:4294967295] 192.168.203.150@tcp LPNI seq info [255:255:8:4294967295]
00000400:00000200:2.0:1788971573.522994:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.523003:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971573.523009:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971573.523014:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63a00.
00000800:00000200:2.0:1788971573.523043:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971573.523052:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971573.523059:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63a00 type 1, nob 320 niov 1 nkiov 1
00000100:00000010:3.0:1788971573.523073:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80.
00000400:00000200:2.0:1788971573.523080:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788971573.523084:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
00000100:00000010:2.0:1788971573.523088:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726cc00.
00000100:00000010:3.0:1788971573.523103:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790700.
00000100:00000010:2.0:1788971573.523106:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c300.
02000000:00000010:3.0:1788971573.523112:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00.
02000000:00000010:2.0:1788971573.523137:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600.
00000400:00000200:2.0:1788971573.523144:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971573.523148:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00.
00000400:00000010:2.0:1788971573.523152:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0.
00000100:00000200:2.0:1788971573.523155:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545984, portal 10
00000400:00000010:2.0:1788971573.523162:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788971573.523164:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189545984, offset 0
00000400:00000200:2.0:1788971573.523169:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971573.523173:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.523176:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971573.523179:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971573.523181:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971573.523183:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971573.523186:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971573.523188:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [256:256:255:4294967295] 192.168.203.150@tcp LPNI seq info [256:256:7:4294967295]
00000400:00000200:2.0:1788971573.523192:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.523197:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971573.523201:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971573.523206:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63b00.
00000800:00000200:2.0:1788971573.523210:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971573.523214:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971573.523217:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971573.523222:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971573.523270:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971573.523279:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63a00.
00000400:00000200:2.0:1788971573.523285:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971573.523289:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971573.523294:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8
00000400:00000010:2.0:1788971573.523297:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788971573.523306:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8790380 x1875872189545728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971573.523316:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003e00.
00000400:00000200:3.0:1788971573.523327:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971573.523332:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688.
00000400:00000010:3.0:1788971573.523341:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788971573.523348:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189545856, portal 10
00000400:00000010:3.0:1788971573.523355:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788971573.523358:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189545856, offset 0
00000400:00000200:3.0:1788971573.523368:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971573.523376:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.523384:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971573.523389:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971573.523393:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971573.523397:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000800:00000200:2.0:1788971573.523402:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971573.523407:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63b00.
00000400:00000200:2.0:1788971573.523410:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971573.523418:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971573.523423:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971573.523501:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [257:257:255:4294967295] 192.168.203.150@tcp LPNI seq info [257:257:7:4294967295]
00000400:00000200:3.0:1788971573.523508:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.523840:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971573.523847:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971573.523853:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300.
00000800:00000200:3.0:1788971573.523860:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971573.523866:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971573.523870:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971573.523880:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.525432:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550
00000400:00000010:2.0:1788971573.525440:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788971573.525454:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8790700 x1875872189545984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971573.525559:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971573.525563:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971573.525568:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971573.525572:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971573.525576:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8
00000400:00000010:2.0:1788971573.525578:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8.
00000100:00000200:2.0:1788971573.525761:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792d80 x1875872189545856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971573.526700:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526711:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971573.526721:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526727:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526733:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526738:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807e00
00000400:00000200:3.0:1788971573.526754:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526761:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd0d [1] + 224
00000400:00000200:3.0:1788971573.526774:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971573.526779:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971573.526794:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8790700 x1875872189545984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971573.526872:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526876:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971573.526881:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526883:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526886:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526889:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807d00
00000400:00000200:3.0:1788971573.526893:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971573.526915:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xcfd [1] + 224
00000400:00000200:3.0:1788971573.527346:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971573.527352:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971573.527362:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8790380 x1875872189545728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971573.527489:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00.
00000400:00000200:2.0:1788971573.527496:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0
00000400:00000010:2.0:1788971573.527499:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0.
00000100:00000200:2.0:1788971573.527506:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8790700 x1875872189545984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971573.527517:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8790700 x1875872189545984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971573.527556:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126a50.
00000400:00000200:2.0:1788971573.527569:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000400:00000010:2.0:1788971573.527571:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971573.527576:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8790380 x1875872189545728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971573.527585:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8790380 x1875872189545728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971573.527603:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600.
02000000:00000010:2.0:1788971573.527611:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00.
00000100:00000010:2.0:1788971573.527616:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790700.
02000000:00000010:2.0:1788971573.527620:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9ec00.
02000000:00000010:2.0:1788971573.527623:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600.
00000100:00000010:2.0:1788971573.527625:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380.
00000400:00000200:2.0:1788971573.529701:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.529712:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971573.529721:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.529726:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.529732:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.529737:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807d80
00000400:00000200:2.0:1788971573.529743:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971573.529751:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd1d [1] + 224
00000400:00000200:2.0:1788971573.529767:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971573.529772:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971573.529789:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792d80 x1875872189545856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971573.530313:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688.
00000400:00000200:3.0:1788971573.530320:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8
00000400:00000010:3.0:1788971573.530324:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788971573.530332:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792d80 x1875872189545856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971573.530342:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792d80 x1875872189545856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971589 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971573.530378:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003e00.
02000000:00000010:3.0:1788971573.530385:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:3.0:1788971573.530390:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80.
00000020:00000010:1.0:1788971574.323472:0:109825:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971574.323560:0:109825:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000100:00000010:3.0:1788971574.546057:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c300.
00000100:00000010:2.0:1788971574.550022:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726cc00.
00000020:00000010:1.0:1788971575.360776:0:109829:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971575.361213:0:109829:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971576.404776:0:109833:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971576.404940:0:109833:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971577.443744:0:109837:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0050.
00000020:00000010:1.0:1788971577.443865:0:109837:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0050.
00000020:00000010:3.0:1788971578.483930:0:109841:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971578.484020:0:109841:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:3.0:1788971578.642057:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80.
02000000:00000010:3.0:1788971578.642090:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
00000100:00000010:3.0:1788971578.642147:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000.
02000000:00000010:3.0:1788971578.642154:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100.
00000100:00000010:3.0:1788971578.642162:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791f80.
02000000:00000010:3.0:1788971578.642167:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00.
02000000:00000010:3.0:1788971578.642206:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003400.
00000400:00000200:3.0:1788971578.642217:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971578.642222:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688.
00000400:00000010:3.0:1788971578.642230:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788971578.642240:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546240, portal 10
00000400:00000010:3.0:1788971578.642261:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788971578.642285:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189546240, offset 0
02000000:00000010:2.0:1788971578.642302:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9ec00.
00000400:00000200:3.0:1788971578.642310:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971578.642324:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.642335:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971578.642341:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971578.642346:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971578.642351:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971578.642354:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971578.642358:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [258:258:256:4294967295] 192.168.203.150@tcp LPNI seq info [258:258:8:4294967295]
00000400:00000200:3.0:1788971578.642364:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.642375:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971578.642382:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971578.642395:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00.
00000800:00000200:3.0:1788971578.642405:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971578.642415:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971578.642422:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971578.642444:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788971578.642451:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391720.
00000100:00000010:3.0:1788971578.642466:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391d20.
00000800:00000200:3.0:1788971578.642609:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971578.642615:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00.
00000400:00000200:3.0:1788971578.642621:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971578.642626:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971578.642633:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8
00000400:00000010:3.0:1788971578.642636:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788971578.642657:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8790000 x1875872189546240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971578.643266:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971578.643278:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126a50.
00000400:00000010:2.0:1788971578.643287:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971578.643295:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546112, portal 25
00000400:00000010:2.0:1788971578.643315:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0.
00000100:00000200:2.0:1788971578.643318:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189546112, offset 0
00000400:00000200:2.0:1788971578.643447:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971578.643457:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971578.643465:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971578.643470:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971578.643474:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971578.643478:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971578.643481:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971578.643485:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [259:259:256:4294967295] 192.168.203.150@tcp LPNI seq info [259:259:8:4294967295]
00000400:00000200:2.0:1788971578.643489:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971578.643497:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971578.643503:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971578.643509:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63b00.
00000800:00000200:2.0:1788971578.643516:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971578.643523:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971578.643528:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971578.643741:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788971578.643768:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600.
00000400:00000200:2.0:1788971578.643773:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971578.643776:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00.
00000400:00000010:2.0:1788971578.643780:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788971578.643783:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546368, portal 10
00000400:00000010:2.0:1788971578.643788:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788971578.643791:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189546368, offset 0
00000400:00000200:2.0:1788971578.643795:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971578.643799:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971578.643803:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971578.643806:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971578.643809:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971578.643811:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971578.643846:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971578.643851:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [260:260:255:4294967295] 192.168.203.150@tcp LPNI seq info [260:260:7:4294967295]
00000400:00000200:2.0:1788971578.643855:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971578.643862:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971578.643870:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971578.643874:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63a00.
00000800:00000200:2.0:1788971578.643880:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971578.643884:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971578.643887:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971578.643894:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971578.643901:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971578.643910:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63b00.
00000400:00000200:3.0:1788971578.643917:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971578.643925:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971578.643942:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0
00000400:00000010:3.0:1788971578.643945:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0.
00000100:00000200:3.0:1788971578.643955:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792d80 x1875872189546112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971578.644440:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971578.644445:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63a00.
00000400:00000200:3.0:1788971578.644450:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971578.644456:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971578.644460:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8
00000400:00000010:3.0:1788971578.644463:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8.
00000100:00000200:3.0:1788971578.644471:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8791f80 x1875872189546368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971578.647493:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.647506:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971578.647516:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.647522:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.647528:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.647533:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807f00
00000400:00000200:3.0:1788971578.647539:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.647546:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd2d [1] + 224
00000400:00000200:3.0:1788971578.647562:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971578.647568:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971578.647585:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8790000 x1875872189546240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971578.647812:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688.
00000400:00000200:2.0:1788971578.647855:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8
00000400:00000010:2.0:1788971578.647859:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8.
00000100:00000200:2.0:1788971578.647867:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8790000 x1875872189546240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971578.647877:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8790000 x1875872189546240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971578.647921:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003400.
02000000:00000010:2.0:1788971578.647929:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100.
00000100:00000010:2.0:1788971578.647934:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000.
00000400:00000200:3.0:1788971578.648795:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.648803:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971578.648810:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.648854:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.648860:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.648865:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887807f80
00000400:00000200:3.0:1788971578.648894:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.648901:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd4d [1] + 224
00000400:00000200:3.0:1788971578.648915:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971578.648919:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971578.649049:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8791f80 x1875872189546368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971578.649086:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.649090:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971578.649095:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.649098:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.649101:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.649174:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887807e80
00000400:00000200:3.0:1788971578.649180:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971578.649186:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xd3d [1] + 224
00000400:00000200:3.0:1788971578.649197:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971578.649201:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971578.649209:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792d80 x1875872189546112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971578.649537:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126a50.
00000400:00000200:2.0:1788971578.649544:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000400:00000010:2.0:1788971578.649547:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971578.649554:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792d80 x1875872189546112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971578.649563:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792d80 x1875872189546112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971578.649586:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00.
00000400:00000200:2.0:1788971578.649590:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550
00000400:00000010:2.0:1788971578.649604:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788971578.649610:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8791f80 x1875872189546368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971578.649617:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8791f80 x1875872189546368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971594 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971578.649641:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9ec00.
02000000:00000010:2.0:1788971578.649657:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:2.0:1788971578.649662:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80.
02000000:00000010:2.0:1788971578.649667:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600.
02000000:00000010:2.0:1788971578.649669:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00.
00000100:00000010:2.0:1788971578.649671:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791f80.
00000020:00000010:0.0:1788971579.516752:0:109846:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971579.516894:0:109846:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000100:00000010:2.0:1788971579.666184:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391720.
00000100:00000010:3.0:1788971579.666777:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391d20.
00000020:00000010:2.0:1788971580.567864:0:109850:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971580.567975:0:109850:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971581.627870:0:109854:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971581.628172:0:109854:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000020:00000010:0.0:1788971582.687635:0:109858:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971582.688465:0:109858:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:3.0:1788971583.719584:0:109862:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971583.719705:0:109862:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:3.0:1788971583.761936:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100.
02000000:00000010:3.0:1788971583.761981:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00.
00000100:00000010:3.0:1788971583.762019:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971583.762040:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00.
00000100:00000010:3.0:1788971583.762047:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793480.
02000000:00000010:3.0:1788971583.762051:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
02000000:00000010:3.0:1788971583.762107:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002a00.
00000400:00000200:3.0:1788971583.762120:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971583.762145:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160.
00000400:00000010:3.0:1788971583.762162:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788971583.762173:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546496, portal 25
00000400:00000010:3.0:1788971583.762180:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48736e8.
00000100:00000200:3.0:1788971583.762184:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189546496, offset 0
00000400:00000200:3.0:1788971583.762196:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971583.762420:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.762433:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971583.762441:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971583.762447:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971583.762453:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971583.762456:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971583.762461:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [261:261:256:4294967295] 192.168.203.150@tcp LPNI seq info [261:261:8:4294967295]
00000400:00000200:3.0:1788971583.762466:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788971583.762474:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9ec00.
00000400:00000200:3.0:1788971583.762479:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788971583.762485:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971583.762491:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000400:00000010:2.0:1788971583.762491:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00.
00000400:00000010:2.0:1788971583.762497:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550.
00000800:00000010:3.0:1788971583.762501:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100.
00000100:00000200:2.0:1788971583.762503:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546624, portal 10
00000400:00000010:2.0:1788971583.762512:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
00000800:00000200:3.0:1788971583.762514:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000100:00000200:2.0:1788971583.762516:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189546624, offset 0
00000400:00000200:2.0:1788971583.762525:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000800:00000200:3.0:1788971583.762530:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:2.0:1788971583.762534:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971583.762538:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971583.762543:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971583.762548:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971583.762551:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971583.762554:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971583.762556:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971583.762561:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [262:262:255:4294967295] 192.168.203.150@tcp LPNI seq info [262:262:7:4294967295]
00000400:00000200:3.0:1788971583.762562:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788971583.762570:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391d20.
00000400:00000200:2.0:1788971583.762579:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788971583.762588:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391d80.
00000400:00000200:2.0:1788971583.762588:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971583.762596:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971583.762601:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63000.
00000800:00000200:2.0:1788971583.762605:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971583.762611:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971583.762624:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63000 type 1, nob 320 niov 1 nkiov 1
02000000:00000010:3.0:1788971583.762625:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002600.
00000400:00000200:3.0:1788971583.762632:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.762636:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971583.762638:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18.
00000400:00000010:3.0:1788971583.762644:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0.
00000100:00000200:3.0:1788971583.762647:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546752, portal 10
00000400:00000010:3.0:1788971583.762652:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8.
00000100:00000200:3.0:1788971583.762655:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189546752, offset 0
00000400:00000200:3.0:1788971583.762662:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971583.762673:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.762677:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971583.762681:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971583.762683:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971583.762686:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971583.762688:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971583.762691:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [263:263:254:4294967295] 192.168.203.150@tcp LPNI seq info [263:263:6:4294967295]
00000400:00000200:3.0:1788971583.762695:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.762702:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971583.762707:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971583.762711:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600.
00000800:00000200:3.0:1788971583.762715:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971583.762721:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971583.762724:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971583.762730:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971583.762731:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971583.762737:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100.
00000400:00000200:2.0:1788971583.762747:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971583.762754:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971583.762762:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48736e8
00000400:00000010:2.0:1788971583.762766:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48736e8.
00000100:00000200:2.0:1788971583.762779:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793100 x1875872189546496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971583.762808:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971583.762810:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63000.
00000400:00000200:2.0:1788971583.762852:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971583.762856:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971583.762859:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000400:00000010:2.0:1788971583.762861:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971583.762866:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792300 x1875872189546624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971583.762953:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971583.762957:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600.
00000400:00000200:2.0:1788971583.762960:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971583.762964:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971583.762967:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8
00000400:00000010:2.0:1788971583.762969:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8.
00000100:00000200:2.0:1788971583.762973:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793480 x1875872189546752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971583.768685:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.768698:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971583.768710:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.768716:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.768723:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.768728:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808000
00000400:00000200:3.0:1788971583.768734:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971583.768741:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xd5d [1] + 224
00000400:00000200:3.0:1788971583.769559:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971583.769569:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971583.769589:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793100 x1875872189546496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971583.770779:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770790:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971583.770800:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770805:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770811:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770852:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808080
00000400:00000200:2.0:1788971583.770858:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770865:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd6d [1] + 224
00000400:00000200:2.0:1788971583.770878:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971583.770883:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971583.770894:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792300 x1875872189546624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971583.770941:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770944:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971583.770949:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770951:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770954:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770956:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808100
00000400:00000200:2.0:1788971583.770959:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971583.770963:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd7d [1] + 224
00000400:00000200:2.0:1788971583.770976:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971583.770979:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971583.770986:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793480 x1875872189546752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971583.771043:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160.
00000400:00000200:2.0:1788971583.771049:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8
00000400:00000010:2.0:1788971583.771053:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8.
00000100:00000200:2.0:1788971583.771060:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793100 x1875872189546496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971583.771248:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793100 x1875872189546496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971583.771280:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18.
00000400:00000200:2.0:1788971583.771283:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0
00000400:00000010:2.0:1788971583.771285:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0.
00000100:00000200:2.0:1788971583.771290:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793480 x1875872189546752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971583.771299:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793480 x1875872189546752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971583.771333:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002a00.
02000000:00000010:2.0:1788971583.771342:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00.
00000100:00000010:2.0:1788971583.771348:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100.
02000000:00000010:2.0:1788971583.771352:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002600.
02000000:00000010:2.0:1788971583.771355:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:2.0:1788971583.771357:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793480.
00000400:00000010:3.0:1788971583.771498:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00.
00000400:00000200:3.0:1788971583.771504:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550
00000400:00000010:3.0:1788971583.771505:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550.
00000100:00000200:3.0:1788971583.771521:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792300 x1875872189546624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971583.771531:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792300 x1875872189546624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971599 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971583.771551:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9ec00.
02000000:00000010:3.0:1788971583.771555:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00.
00000100:00000010:3.0:1788971583.771559:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300.
00000020:00000010:0.0:1788971584.782052:0:109867:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971584.782539:0:109867:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000100:00000010:2.0:1788971584.785909:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391d20.
00000100:00000010:3.0:1788971584.786000:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391d80.
00000020:00000010:0.0:1788971585.831192:0:109871:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971585.831291:0:109871:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:1.0:1788971586.858449:0:109875:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971586.858571:0:109875:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971587.899284:0:109879:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971587.899390:0:109879:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000100:00000010:1.0:1788971588.882072:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788971588.882094:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0ff00.
00000100:00000010:1.0:1788971588.882122:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80.
02000000:00000010:1.0:1788971588.882129:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00.
00000100:00000010:1.0:1788971588.882135:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700.
02000000:00000010:1.0:1788971588.882138:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f900.
02000000:00000010:1.0:1788971588.882334:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17af200.
00000400:00000200:1.0:1788971588.882355:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971588.882359:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000010:1.0:1788971588.882366:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971588.882372:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189546880, portal 25
00000400:00000010:1.0:1788971588.882377:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0.
00000100:00000200:1.0:1788971588.882381:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189546880, offset 0
00000400:00000200:1.0:1788971588.882390:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971588.882398:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971588.882407:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971588.882413:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971588.882418:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971588.882422:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971588.882425:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971588.882429:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [264:264:256:4294967295] 192.168.203.150@tcp LPNI seq info [264:264:8:4294967295]
00000400:00000200:1.0:1788971588.882434:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971588.882443:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971588.882448:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971588.882453:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f800.
00000800:00000200:1.0:1788971588.882460:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971588.882468:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971588.882473:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971588.882489:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:1.0:1788971588.882494:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467ba0.
00000100:00000010:1.0:1788971588.882504:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467f00.
02000000:00000010:1.0:1788971588.882532:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17ae200.
00000400:00000200:1.0:1788971588.882536:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971588.882538:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738.
00000400:00000010:1.0:1788971588.882542:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880.
00000800:00000200:2.0:1788971588.882545:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000100:00000200:1.0:1788971588.882546:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547136, portal 10
00000800:00000010:2.0:1788971588.882553:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f800.
00000400:00000010:1.0:1788971588.882554:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525110.
00000100:00000200:1.0:1788971588.882558:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189547136, offset 0
00000400:00000200:2.0:1788971588.882559:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971588.882568:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:1.0:1788971588.882572:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
02000000:00000010:0.0:1788971588.885596:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5600.
00000400:00000200:0.0:1788971588.885611:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971588.885616:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971588.885622:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971588.885630:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547008, portal 10
00000400:00000010:0.0:1788971588.885637:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8dd0.
00000400:00000200:3.0:1788971588.885875:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971588.885884:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971588.885891:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971588.885895:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971588.885899:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971588.885903:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808180
00000400:00000200:3.0:1788971588.885908:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971588.885914:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xd8d [1] + 224
00000400:00000200:3.0:1788971588.886141:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971588.886146:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971588.886160:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189546880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971588.886795:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0
00000400:00000010:2.0:1788971588.886799:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0.
00000100:00000200:2.0:1788971588.886809:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189546880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971604 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971588.886921:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189547008, offset 0
00000400:00000200:0.0:1788971588.886931:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971588.886938:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971588.886945:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971588.886967:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971588.886972:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971588.886975:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971588.886978:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971588.886982:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [265:265:256:4294967295] 192.168.203.150@tcp LPNI seq info [265:265:8:4294967295]
00000400:00000200:0.0:1788971588.886988:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971588.886999:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971588.887005:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971588.887012:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ee00.
00000800:00000200:0.0:1788971588.887019:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971588.887026:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971588.887031:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971588.887051:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971588.888336:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971588.888343:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ee00.
00000400:00000200:2.0:1788971588.888349:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971588.888357:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971588.888363:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8dd0
00000400:00000010:2.0:1788971588.888376:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8dd0.
00000100:00000200:2.0:1788971588.888386:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f2d80 x1875872189547008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788971588.888660:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971588.888667:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971588.888672:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971588.888676:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971588.888680:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971588.888683:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971588.888686:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [266:266:256:4294967295] 192.168.203.150@tcp LPNI seq info [266:266:8:4294967295]
00000400:00000200:1.0:1788971588.888691:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971588.888700:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971588.888707:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971588.888713:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f000.
00000800:00000200:1.0:1788971588.888719:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971588.888727:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971588.888731:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971588.888755:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971588.888775:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000200:1.0:1788971588.888779:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220
00000400:00000010:1.0:1788971588.888781:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971588.888788:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189546880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971588.888799:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189546880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971588.888850:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17af200.
02000000:00000010:1.0:1788971588.889249:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0ff00.
00000100:00000010:1.0:1788971588.889254:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
00000800:00000200:2.0:1788971588.889322:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971588.889326:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f000.
00000400:00000200:2.0:1788971588.889329:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971588.889334:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971588.889338:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525110
00000400:00000010:2.0:1788971588.889340:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525110.
00000100:00000200:2.0:1788971588.889347:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f0700 x1875872189547136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971588.890774:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.890782:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971588.890789:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.890793:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.890798:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.890801:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808200
00000400:00000200:2.0:1788971588.890806:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.891010:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd9d [1] + 224
00000400:00000200:2.0:1788971588.891027:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971588.891031:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971588.891041:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f2d80 x1875872189547008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971588.893010:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.893019:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971588.893027:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.893032:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.893037:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.893041:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808280
00000400:00000200:2.0:1788971588.893046:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971588.893067:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xd95 [1] + 224
00000400:00000200:2.0:1788971588.893464:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971588.893471:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971588.893483:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f0700 x1875872189547136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971588.896057:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971588.896067:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971588.896070:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971588.896082:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f2d80 x1875872189547008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971588.896094:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f2d80 x1875872189547008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971588.896130:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5600.
02000000:00000010:0.0:1788971588.896137:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00.
00000100:00000010:0.0:1788971588.896141:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80.
00000400:00000010:0.0:1788971588.896182:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738.
00000400:00000200:0.0:1788971588.896185:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880
00000400:00000010:0.0:1788971588.896187:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880.
00000100:00000200:0.0:1788971588.896194:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f0700 x1875872189547136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971588.896203:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f0700 x1875872189547136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971604 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971588.896242:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17ae200.
02000000:00000010:0.0:1788971588.896245:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f900.
00000100:00000010:0.0:1788971588.896248:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700.
00000020:00000010:1.0:1788971588.947530:0:109884:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971588.948573:0:109884:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000100:00000010:2.0:1788971589.905907:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467ba0.
00000100:00000010:3.0:1788971589.907694:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467f00.
00000020:00000010:1.0:1788971589.983133:0:109888:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971589.983307:0:109888:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000020:00000010:0.0:1788971591.015206:0:109892:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971591.015309:0:109892:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:1.0:1788971592.050704:0:109896:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971592.050803:0:109896:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000020:00000010:3.0:1788971593.100495:0:109900:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971593.100607:0:109900:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:1.0:1788971594.005703:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788971594.005725:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0ff00.
00000100:00000010:1.0:1788971594.005753:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80.
02000000:00000010:1.0:1788971594.005758:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fb00.
00000100:00000010:1.0:1788971594.005764:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3b80.
02000000:00000010:1.0:1788971594.005768:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f600.
02000000:00000010:0.0:1788971594.006006:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4000.
00000400:00000200:0.0:1788971594.006015:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971594.006019:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971594.006025:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971594.006031:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547264, portal 25
00000400:00000010:0.0:1788971594.006036:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58.
00000100:00000200:0.0:1788971594.006039:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189547264, offset 0
00000400:00000200:0.0:1788971594.006047:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971594.006055:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971594.006063:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971594.006067:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971594.006071:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971594.006075:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971594.006077:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971594.006081:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [267:267:256:4294967295] 192.168.203.150@tcp LPNI seq info [267:267:8:4294967295]
00000400:00000200:0.0:1788971594.006085:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971594.006094:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971594.006100:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971594.006105:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1e900.
00000800:00000200:0.0:1788971594.006111:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971594.006118:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971594.006122:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1e900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971594.006143:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971594.006149:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c59c0.
00000100:00000010:0.0:1788971594.006176:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c57e0.
02000000:00000010:0.0:1788971594.006187:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5000.
00000400:00000200:0.0:1788971594.006192:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971594.006194:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971594.006198:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8330.
00000100:00000200:0.0:1788971594.006201:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547520, portal 10
00000400:00000010:0.0:1788971594.006206:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8aa0.
00000100:00000200:0.0:1788971594.006208:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189547520, offset 0
00000400:00000200:0.0:1788971594.006212:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971594.006216:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971594.006219:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971594.006222:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971594.006225:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971594.006227:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971594.006229:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971594.006232:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [268:268:255:4294967295] 192.168.203.150@tcp LPNI seq info [268:268:7:4294967295]
00000400:00000200:0.0:1788971594.006236:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971594.006242:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971594.006245:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971594.006249:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ec00.
00000800:00000200:0.0:1788971594.006252:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971594.006256:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971594.006258:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ec00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971594.006263:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971594.006280:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5a00.
00000400:00000200:0.0:1788971594.006283:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971594.006286:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971594.006289:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8f68.
00000100:00000200:0.0:1788971594.006291:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547392, portal 10
00000400:00000010:0.0:1788971594.006295:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a86e8.
00000100:00000200:0.0:1788971594.006297:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189547392, offset 0
00000400:00000200:0.0:1788971594.006307:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971594.006310:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971594.006313:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971594.006316:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971594.006318:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971594.006320:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971594.006322:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971594.006325:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [269:269:254:4294967295] 192.168.203.150@tcp LPNI seq info [269:269:6:4294967295]
00000400:00000200:0.0:1788971594.006328:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971594.006332:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971594.006336:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971594.006340:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7d1ee00.
00000800:00000200:0.0:1788971594.006344:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971594.006348:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971594.006350:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7d1ee00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971594.006354:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971594.006466:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971594.006470:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1e900.
00000400:00000200:2.0:1788971594.006477:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971594.006482:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971594.006487:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58
00000400:00000010:2.0:1788971594.006489:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58.
00000100:00000200:2.0:1788971594.006497:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189547264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971594.006520:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971594.006522:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ec00.
00000400:00000200:2.0:1788971594.006525:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971594.006527:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971594.006529:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8aa0
00000400:00000010:2.0:1788971594.006531:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8aa0.
00000100:00000200:2.0:1788971594.006535:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3b80 x1875872189547520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971594.006611:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971594.006614:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7d1ee00.
00000400:00000200:2.0:1788971594.006617:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971594.006621:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971594.006623:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a86e8
00000400:00000010:2.0:1788971594.006625:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a86e8.
00000100:00000200:2.0:1788971594.006630:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f1f80 x1875872189547392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971594.011370:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011405:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971594.011417:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011423:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011429:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011433:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808400
00000400:00000200:2.0:1788971594.011440:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011448:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xdad [1] + 224
00000400:00000200:2.0:1788971594.011463:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971594.011468:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971594.011486:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3b80 x1875872189547520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971594.011750:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011755:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971594.011760:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011762:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011766:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011769:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808380
00000400:00000200:2.0:1788971594.011773:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011777:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xdb5 [1] + 224
00000400:00000200:2.0:1788971594.011783:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971594.011787:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971594.011796:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f1f80 x1875872189547392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971594.011810:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011843:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971594.011847:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011850:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011853:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011856:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808300
00000400:00000200:2.0:1788971594.011859:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971594.011870:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xda5 [1] + 224
00000400:00000200:2.0:1788971594.011881:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971594.011885:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971594.011891:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189547264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971594.012015:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971594.012022:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8330
00000400:00000010:0.0:1788971594.012025:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8330.
00000100:00000200:0.0:1788971594.012033:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3b80 x1875872189547520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971594.012043:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3b80 x1875872189547520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971594.012084:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5000.
02000000:00000010:0.0:1788971594.012153:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f600.
00000100:00000010:0.0:1788971594.012161:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3b80.
00000400:00000010:0.0:1788971594.012169:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971594.012172:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8f68
00000400:00000010:0.0:1788971594.012173:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8f68.
00000100:00000200:0.0:1788971594.012178:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f1f80 x1875872189547392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971594.012259:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f1f80 x1875872189547392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971594.012276:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971594.012279:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971594.012281:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971594.012285:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189547264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971594.012294:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189547264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971610 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971594.012304:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5a00.
02000000:00000010:0.0:1788971594.012307:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fb00.
00000100:00000010:0.0:1788971594.012309:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80.
02000000:00000010:0.0:1788971594.012313:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4000.
02000000:00000010:0.0:1788971594.012315:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0ff00.
00000100:00000010:0.0:1788971594.012317:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
00000020:00000010:3.0:1788971594.153648:0:109904:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971594.153740:0:109904:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:2.0:1788971595.026555:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c59c0.
00000100:00000010:3.0:1788971595.027142:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c57e0.
00000020:00000010:0.0:1788971595.190233:0:109908:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971595.190344:0:109908:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:1.0:1788971596.281771:0:109913:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971596.285176:0:109913:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000020:00000010:0.0:1788971597.320235:0:109917:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971597.320343:0:109917:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:3.0:1788971598.397400:0:109921:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971598.400927:0:109921:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000100:00000010:3.0:1788971599.122899:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971599.122932:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00.
00000100:00000010:3.0:1788971599.123002:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80.
02000000:00000010:3.0:1788971599.123009:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300.
00000100:00000010:3.0:1788971599.123016:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800.
02000000:00000010:3.0:1788971599.123022:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
02000000:00000010:2.0:1788971599.123588:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600.
00000400:00000200:2.0:1788971599.123603:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971599.123608:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126a50.
00000400:00000010:2.0:1788971599.123617:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971599.123626:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547648, portal 25
00000400:00000010:2.0:1788971599.123633:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000.
00000100:00000200:2.0:1788971599.123636:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189547648, offset 0
00000400:00000200:2.0:1788971599.123649:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971599.123659:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.123669:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971599.123676:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971599.123680:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971599.123684:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971599.123688:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971599.123693:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [270:270:256:4294967295] 192.168.203.150@tcp LPNI seq info [270:270:8:4294967295]
00000400:00000200:2.0:1788971599.123698:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.123708:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971599.123715:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971599.123721:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63000.
00000800:00000200:2.0:1788971599.123729:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971599.123737:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971599.123743:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971599.123768:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971599.123775:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726cc00.
00000100:00000010:2.0:1788971599.123798:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c300.
02000000:00000010:2.0:1788971599.123810:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9fa00.
00000400:00000200:2.0:1788971599.123858:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971599.123862:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126948.
00000400:00000010:2.0:1788971599.123866:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440.
00000100:00000200:2.0:1788971599.123869:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547904, portal 10
00000400:00000010:2.0:1788971599.123876:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880.
00000100:00000200:2.0:1788971599.123878:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189547904, offset 0
00000400:00000200:2.0:1788971599.123883:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971599.123886:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.123890:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971599.123892:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971599.123895:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971599.123897:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971599.123899:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971599.123902:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [271:271:255:4294967295] 192.168.203.150@tcp LPNI seq info [271:271:7:4294967295]
00000400:00000200:2.0:1788971599.123906:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.123911:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971599.123915:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971599.123919:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63800.
00000800:00000200:2.0:1788971599.123923:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971599.123926:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971599.123929:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971599.123934:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788971599.123956:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000.
00000400:00000200:2.0:1788971599.123959:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971599.123962:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f20.
00000400:00000010:2.0:1788971599.123965:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48.
00000100:00000200:2.0:1788971599.123967:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189547776, portal 10
00000400:00000010:2.0:1788971599.123971:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b990.
00000100:00000200:2.0:1788971599.123973:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189547776, offset 0
00000400:00000200:2.0:1788971599.123985:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971599.123989:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.123992:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971599.123995:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971599.123997:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971599.123999:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971599.124001:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971599.124003:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [272:272:254:4294967295] 192.168.203.150@tcp LPNI seq info [272:272:6:4294967295]
00000400:00000200:2.0:1788971599.124007:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.124012:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971599.124017:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971599.124020:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00.
00000800:00000200:2.0:1788971599.124028:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971599.124033:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971599.124036:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971599.124041:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971599.124211:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971599.124216:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63000.
00000400:00000200:3.0:1788971599.124224:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971599.124230:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971599.124236:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000
00000400:00000010:3.0:1788971599.124239:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000.
00000100:00000200:3.0:1788971599.124253:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792300 x1875872189547648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971599.124280:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971599.124283:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63800.
00000400:00000200:3.0:1788971599.124286:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971599.124289:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971599.124292:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880
00000400:00000010:3.0:1788971599.124293:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880.
00000100:00000200:3.0:1788971599.124297:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793800 x1875872189547904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971599.124383:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971599.124387:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00.
00000400:00000200:3.0:1788971599.124390:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971599.124393:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971599.124396:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b990
00000400:00000010:3.0:1788971599.124398:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b990.
00000100:00000200:3.0:1788971599.124403:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793b80 x1875872189547776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971599.132524:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132538:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971599.132551:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132556:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132562:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132567:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808580
00000400:00000200:2.0:1788971599.132574:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132580:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xdcd [1] + 224
00000400:00000200:2.0:1788971599.132594:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971599.132599:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971599.132617:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793800 x1875872189547904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971599.132655:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132659:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971599.132682:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132685:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132688:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132691:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808500
00000400:00000200:2.0:1788971599.132695:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132699:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xddd [1] + 224
00000400:00000200:2.0:1788971599.132706:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971599.132709:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971599.132716:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793b80 x1875872189547776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971599.132731:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132734:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971599.132738:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132740:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132743:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132745:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808480
00000400:00000200:2.0:1788971599.132749:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971599.132753:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xdbd [1] + 224
00000400:00000200:2.0:1788971599.132765:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971599.132768:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971599.132775:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792300 x1875872189547648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971599.132812:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f20.
00000400:00000200:2.0:1788971599.132848:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48
00000400:00000010:2.0:1788971599.132851:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48.
00000100:00000200:2.0:1788971599.132858:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793b80 x1875872189547776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971599.132866:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793b80 x1875872189547776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971599.132918:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126a50.
00000400:00000200:2.0:1788971599.132921:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000400:00000010:2.0:1788971599.132923:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971599.132928:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792300 x1875872189547648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971599.132936:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792300 x1875872189547648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971599.132946:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126948.
00000400:00000200:2.0:1788971599.132948:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440
00000400:00000010:2.0:1788971599.132950:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440.
00000100:00000200:2.0:1788971599.132954:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793800 x1875872189547904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971599.132961:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793800 x1875872189547904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971615 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971599.132979:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000.
02000000:00000010:2.0:1788971599.132987:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300.
00000100:00000010:2.0:1788971599.132993:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80.
02000000:00000010:2.0:1788971599.132997:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600.
02000000:00000010:2.0:1788971599.132999:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00.
00000100:00000010:2.0:1788971599.133001:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300.
02000000:00000010:2.0:1788971599.133005:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9fa00.
02000000:00000010:2.0:1788971599.133007:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:2.0:1788971599.133009:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800.
00000020:00000010:1.0:1788971599.446676:0:109925:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971599.446973:0:109925:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000100:00000010:2.0:1788971600.145953:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726cc00.
00000100:00000010:3.0:1788971600.147620:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c300.
00000020:00000010:0.0:1788971600.497804:0:109929:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971600.497945:0:109929:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:3.0:1788971601.547208:0:109933:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c59e0.
00000020:00000010:3.0:1788971601.547311:0:109933:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c59e0.
00000020:00000010:0.0:1788971602.623563:0:109937:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971602.623685:0:109937:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:1.0:1788971603.686467:0:109942:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971603.686573:0:109942:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000100:00000010:3.0:1788971604.241974:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800.
02000000:00000010:3.0:1788971604.241999:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00.
00000100:00000010:3.0:1788971604.242036:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971604.242042:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600.
00000100:00000010:3.0:1788971604.242047:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80.
02000000:00000010:3.0:1788971604.242050:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100.
02000000:00000010:2.0:1788971604.242125:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000.
00000400:00000200:2.0:1788971604.242136:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971604.242141:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126948.
00000400:00000010:2.0:1788971604.242148:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440.
00000100:00000200:2.0:1788971604.242156:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548032, portal 25
00000400:00000010:2.0:1788971604.242187:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788971604.242190:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189548032, offset 0
00000400:00000200:2.0:1788971604.242200:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971604.242209:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.242218:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971604.242223:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971604.242227:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971604.242230:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971604.242233:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971604.242236:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [273:273:256:4294967295] 192.168.203.150@tcp LPNI seq info [273:273:8:4294967295]
00000400:00000200:2.0:1788971604.242240:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.242249:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971604.242256:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971604.242261:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00.
00000800:00000200:2.0:1788971604.242268:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971604.242276:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971604.242280:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971604.242299:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971604.242304:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726cc00.
00000100:00000010:2.0:1788971604.242314:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726cd80.
02000000:00000010:2.0:1788971604.242324:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9fa00.
00000400:00000200:2.0:1788971604.242327:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971604.242330:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126a50.
00000400:00000010:2.0:1788971604.242333:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48.
00000100:00000200:2.0:1788971604.242336:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548288, portal 10
00000400:00000010:2.0:1788971604.242340:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8.
00000100:00000200:2.0:1788971604.242342:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189548288, offset 0
00000400:00000200:2.0:1788971604.242347:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971604.242351:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.242355:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971604.242357:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971604.242368:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971604.242370:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971604.242372:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971604.242374:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [274:274:255:4294967295] 192.168.203.150@tcp LPNI seq info [274:274:7:4294967295]
00000400:00000200:2.0:1788971604.242378:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.242383:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971604.242388:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971604.242391:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63800.
00000800:00000200:2.0:1788971604.242395:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971604.242399:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971604.242402:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971604.242407:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788971604.242425:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9ee00.
00000400:00000200:2.0:1788971604.242429:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971604.242431:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f20.
00000400:00000010:2.0:1788971604.242434:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220.
00000100:00000200:2.0:1788971604.242437:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548160, portal 10
00000400:00000010:2.0:1788971604.242441:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58.
00000100:00000200:2.0:1788971604.242444:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189548160, offset 0
00000400:00000200:2.0:1788971604.242448:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971604.242452:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.242455:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971604.242457:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971604.242459:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971604.242461:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971604.242463:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971604.242466:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [275:275:254:4294967295] 192.168.203.150@tcp LPNI seq info [275:275:6:4294967295]
00000400:00000200:2.0:1788971604.242480:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.242485:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971604.242489:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971604.242498:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63000.
00000800:00000200:2.0:1788971604.242502:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971604.242506:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971604.242508:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971604.242513:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971604.243523:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971604.243530:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00.
00000400:00000200:3.0:1788971604.243537:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971604.243543:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971604.243549:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000400:00000010:3.0:1788971604.243552:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000100:00000200:3.0:1788971604.243562:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793800 x1875872189548032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971604.243582:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971604.243593:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63800.
00000400:00000200:3.0:1788971604.243596:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971604.243600:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971604.243602:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8
00000400:00000010:3.0:1788971604.243723:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8.
00000100:00000200:3.0:1788971604.243731:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793b80 x1875872189548288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971604.243803:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971604.243806:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63000.
00000400:00000200:3.0:1788971604.243809:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971604.243845:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971604.243849:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58
00000400:00000010:3.0:1788971604.243851:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58.
00000100:00000200:3.0:1788971604.243856:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792300 x1875872189548160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971604.251356:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251365:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971604.251374:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251379:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251383:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251387:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808700
00000400:00000200:2.0:1788971604.251393:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251399:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xdfd [1] + 224
00000400:00000200:2.0:1788971604.251409:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971604.251413:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971604.251426:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793b80 x1875872189548288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971604.251475:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251479:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971604.251483:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251485:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251488:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251491:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808680
00000400:00000200:2.0:1788971604.251494:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251498:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe0d [1] + 224
00000400:00000200:2.0:1788971604.251505:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971604.251508:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971604.251526:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792300 x1875872189548160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971604.251562:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251565:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971604.251568:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251571:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251574:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251576:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808600
00000400:00000200:2.0:1788971604.251579:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971604.251583:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xded [1] + 224
00000400:00000200:2.0:1788971604.251602:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971604.251613:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971604.251619:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793800 x1875872189548032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971604.252031:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f20.
00000400:00000200:3.0:1788971604.252042:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220
00000400:00000010:3.0:1788971604.252045:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220.
00000100:00000200:3.0:1788971604.252051:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792300 x1875872189548160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971604.252060:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792300 x1875872189548160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971604.252088:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126948.
00000400:00000200:3.0:1788971604.252091:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440
00000400:00000010:3.0:1788971604.252093:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440.
00000100:00000200:3.0:1788971604.252098:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793800 x1875872189548032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971604.252105:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793800 x1875872189548032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971604.252115:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126a50.
00000400:00000200:3.0:1788971604.252117:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48
00000400:00000010:3.0:1788971604.252119:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48.
00000100:00000200:3.0:1788971604.252124:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793b80 x1875872189548288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971604.252131:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793b80 x1875872189548288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971620 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971604.252146:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9ee00.
02000000:00000010:3.0:1788971604.252151:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600.
00000100:00000010:3.0:1788971604.252156:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971604.252159:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000.
02000000:00000010:3.0:1788971604.252161:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00.
00000100:00000010:3.0:1788971604.252163:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800.
02000000:00000010:3.0:1788971604.252165:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9fa00.
02000000:00000010:3.0:1788971604.252167:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100.
00000100:00000010:3.0:1788971604.252169:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80.
00000020:00000010:2.0:1788971604.750602:0:109946:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971604.750682:0:109946:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000100:00000010:3.0:1788971605.266248:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726cd80.
00000100:00000010:2.0:1788971605.266364:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726cc00.
00000020:00000010:0.0:1788971605.798712:0:109950:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971605.798800:0:109950:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:1.0:1788971606.836234:0:109954:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971606.836670:0:109954:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000020:00000010:0.0:1788971607.879854:0:109958:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971607.879953:0:109958:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000020:00000010:1.0:1788971608.913099:0:109962:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971608.913180:0:109962:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000100:00000010:2.0:1788971609.363334:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00.
02000000:00000010:2.0:1788971609.363358:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63a00.
00000100:00000010:2.0:1788971609.363398:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788971609.363403:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63b00.
00000100:00000010:2.0:1788971609.363410:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c380.
02000000:00000010:2.0:1788971609.363415:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63000.
02000000:00000010:3.0:1788971609.363485:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002a00.
00000400:00000200:3.0:1788971609.363496:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971609.363501:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10.
00000400:00000010:3.0:1788971609.363508:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873880.
00000100:00000200:3.0:1788971609.363515:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548416, portal 25
00000400:00000010:3.0:1788971609.363520:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788971609.363523:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189548416, offset 0
00000400:00000200:3.0:1788971609.363533:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971609.363541:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.363549:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971609.363555:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971609.363559:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971609.363562:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971609.363565:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971609.363569:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [276:276:256:4294967295] 192.168.203.150@tcp LPNI seq info [276:276:8:4294967295]
00000400:00000200:3.0:1788971609.363573:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.363583:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971609.363589:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971609.363594:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100.
00000800:00000200:3.0:1788971609.363601:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971609.363607:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971609.363612:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971609.363626:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788971609.363632:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d13915a0.
00000100:00000010:3.0:1788971609.363642:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391600.
02000000:00000010:2.0:1788971609.363702:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000.
00000400:00000200:2.0:1788971609.363709:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971609.363712:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126318.
00000400:00000010:2.0:1788971609.363716:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971609.363719:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548544, portal 10
00000400:00000010:2.0:1788971609.363724:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971609.363726:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189548544, offset 0
00000400:00000200:2.0:1788971609.363732:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971609.363750:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971609.363755:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971609.363758:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971609.363760:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971609.363763:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971609.363765:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971609.363767:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [277:277:255:4294967295] 192.168.203.150@tcp LPNI seq info [277:277:7:4294967295]
00000400:00000200:2.0:1788971609.363771:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971609.363778:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971609.363782:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971609.363786:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63800.
00000800:00000200:2.0:1788971609.363790:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971609.363795:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971609.363797:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971609.363803:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788971609.365962:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002c00.
00000400:00000200:3.0:1788971609.365975:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971609.365980:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68.
00000400:00000010:3.0:1788971609.365987:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788971609.365993:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548672, portal 10
00000400:00000010:3.0:1788971609.366000:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873c38.
00000100:00000200:3.0:1788971609.366015:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189548672, offset 0
00000400:00000200:3.0:1788971609.366023:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971609.366030:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.366037:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971609.366042:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971609.366045:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971609.366049:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971609.366051:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971609.366054:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [278:278:254:4294967295] 192.168.203.150@tcp LPNI seq info [278:278:6:4294967295]
00000400:00000200:3.0:1788971609.366057:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.366064:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971609.366069:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971609.366073:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00.
00000800:00000200:3.0:1788971609.366078:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971609.366085:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971609.366089:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971609.366096:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971609.366155:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971609.366158:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100.
00000400:00000200:3.0:1788971609.366162:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971609.366167:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971609.366872:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873660
00000400:00000010:3.0:1788971609.366878:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788971609.366951:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840dc00 x1875872189548416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971609.366981:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971609.366984:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63800.
00000400:00000200:3.0:1788971609.366988:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971609.366991:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971609.366995:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198
00000400:00000010:3.0:1788971609.366996:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198.
00000100:00000200:3.0:1788971609.367009:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ed80 x1875872189548544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971609.367580:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971609.367584:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00.
00000400:00000200:3.0:1788971609.367587:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971609.367591:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971609.367596:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873c38
00000400:00000010:3.0:1788971609.367598:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873c38.
00000100:00000200:3.0:1788971609.367604:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840c380 x1875872189548672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971609.370146:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.370155:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971609.370164:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.370168:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.370376:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.370382:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808800
00000400:00000200:3.0:1788971609.370388:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.370394:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe2d [1] + 224
00000400:00000200:3.0:1788971609.370406:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971609.370410:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971609.370423:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ed80 x1875872189548544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971609.370455:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.370459:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971609.370464:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971609.370464:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126318.
00000400:00000200:3.0:1788971609.370469:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971609.370471:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18
00000400:00000200:3.0:1788971609.370474:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971609.370476:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18.
00000400:00000200:3.0:1788971609.370481:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808880
00000400:00000200:3.0:1788971609.370486:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000200:2.0:1788971609.370489:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ed80 x1875872189548544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971609.370493:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe3d [1] + 224
00000100:00000200:2.0:1788971609.370503:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ed80 x1875872189548544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971609.370521:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971609.370526:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971609.370534:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840c380 x1875872189548672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971609.370545:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000.
02000000:00000010:2.0:1788971609.370553:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63b00.
00000100:00000010:2.0:1788971609.370557:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80.
00000400:00000010:3.0:1788971609.370570:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68.
00000400:00000200:3.0:1788971609.370576:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908
00000400:00000010:3.0:1788971609.370578:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788971609.370583:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840c380 x1875872189548672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971609.370593:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840c380 x1875872189548672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971609.370609:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002c00.
02000000:00000010:3.0:1788971609.370613:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63000.
00000100:00000010:3.0:1788971609.370616:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c380.
00000400:00000200:3.0:1788971609.374169:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.374178:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971609.374186:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.374189:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.374194:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.374197:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808780
00000400:00000200:3.0:1788971609.374202:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971609.374207:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xe1d [1] + 224
00000400:00000200:3.0:1788971609.374478:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971609.374484:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971609.374499:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840dc00 x1875872189548416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971609.374588:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10.
00000400:00000200:2.0:1788971609.374594:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873880
00000400:00000010:2.0:1788971609.374596:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873880.
00000100:00000200:2.0:1788971609.374603:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840dc00 x1875872189548416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971609.374629:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840dc00 x1875872189548416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971625 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971609.374653:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002a00.
02000000:00000010:2.0:1788971609.374659:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63a00.
00000100:00000010:2.0:1788971609.374664:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00.
00000020:00000010:0.0:1788971609.950510:0:109966:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780790.
00000020:00000010:0.0:1788971609.950597:0:109966:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780790.
00000100:00000010:2.0:1788971610.385962:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d13915a0.
00000100:00000010:3.0:1788971610.386698:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391600.
00000020:00000010:1.0:1788971611.001643:0:109971:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0aa0.
00000020:00000010:1.0:1788971611.001743:0:109971:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0aa0.
00000020:00000010:2.0:1788971611.019672:0:109972:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed70.
00000020:00000010:2.0:1788971611.019763:0:109972:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed70.
00000100:00000010:3.0:1788971614.482345:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80.
02000000:00000010:3.0:1788971614.482377:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600.
02000000:00000010:2.0:1788971614.482480:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1342c00.
00000400:00000200:2.0:1788971614.482493:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971614.482499:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126318.
00000400:00000010:2.0:1788971614.482506:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971614.482516:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548800, portal 25
00000400:00000010:2.0:1788971614.482522:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971614.482525:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189548800, offset 0
00000400:00000200:2.0:1788971614.482537:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971614.482547:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.482557:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971614.482564:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971614.482569:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971614.482574:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971614.482577:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971614.482581:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [279:279:256:4294967295] 192.168.203.150@tcp LPNI seq info [279:279:8:4294967295]
00000400:00000200:2.0:1788971614.482586:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.482597:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971614.482603:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971614.482610:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63a00.
00000800:00000200:2.0:1788971614.482618:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971614.482628:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971614.482640:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971614.482659:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971614.482665:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c840.
00000100:00000010:2.0:1788971614.482678:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726ca80.
00000100:00000010:3.0:1788971614.482721:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800.
02000000:00000010:3.0:1788971614.482729:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
02000000:00000010:2.0:1788971614.485224:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343e00.
00000400:00000200:2.0:1788971614.485246:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971614.485253:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126e70.
00000400:00000010:2.0:1788971614.485261:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b770.
00000100:00000200:2.0:1788971614.485269:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189548928, portal 10
00000400:00000010:2.0:1788971614.485278:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971614.485281:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189548928, offset 0
00000400:00000200:2.0:1788971614.485292:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971614.485301:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.485310:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971614.485316:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971614.485333:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971614.485337:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971614.485340:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971614.485344:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [280:280:255:4294967295] 192.168.203.150@tcp LPNI seq info [280:280:7:4294967295]
00000400:00000200:2.0:1788971614.485348:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.485359:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971614.485365:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971614.485371:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63b00.
00000800:00000200:2.0:1788971614.485379:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971614.485387:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971614.485406:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971614.485416:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971614.485491:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971614.485495:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63a00.
00000400:00000200:2.0:1788971614.485504:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971614.485509:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971614.485533:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:2.0:1788971614.485536:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971614.485548:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793b80 x1875872189548800/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971614.485648:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971614.485652:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63b00.
00000400:00000200:2.0:1788971614.485655:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971614.485659:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971614.485661:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0
00000400:00000010:2.0:1788971614.485663:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971614.485668:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793800 x1875872189548928/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788971614.486631:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971614.486652:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00.
02000000:00000010:2.0:1788971614.486715:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343000.
00000400:00000200:2.0:1788971614.486723:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971614.486727:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580.
00000400:00000010:2.0:1788971614.486733:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971614.486739:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549056, portal 10
00000400:00000010:2.0:1788971614.487223:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971614.487228:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189549056, offset 0
00000400:00000200:2.0:1788971614.487239:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971614.487248:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.487256:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971614.487261:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971614.487265:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971614.487269:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971614.487272:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971614.487276:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [281:281:256:4294967295] 192.168.203.150@tcp LPNI seq info [281:281:8:4294967295]
00000400:00000200:2.0:1788971614.487280:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.487289:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971614.487295:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971614.487301:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63b00.
00000800:00000200:2.0:1788971614.487384:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971614.487392:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971614.487406:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971614.487426:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971614.487575:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971614.487580:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63b00.
00000400:00000200:3.0:1788971614.487584:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971614.487589:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971614.487595:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:3.0:1788971614.487597:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:3.0:1788971614.487606:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792300 x1875872189549056/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971614.491476:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491488:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971614.491499:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491505:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491512:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491517:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808980
00000400:00000200:2.0:1788971614.491523:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491532:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe5d [1] + 224
00000400:00000200:2.0:1788971614.491547:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971614.491552:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971614.491570:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793800 x1875872189548928/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971614.491609:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491613:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971614.491617:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491620:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491622:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491624:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808a00
00000400:00000200:2.0:1788971614.491628:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491632:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe6d [1] + 224
00000400:00000200:2.0:1788971614.491638:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971614.491641:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971614.491647:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792300 x1875872189549056/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971614.491660:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491686:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971614.491690:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491693:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491696:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491698:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808900
00000400:00000200:2.0:1788971614.491702:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971614.491706:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xe4d [1] + 224
00000400:00000200:2.0:1788971614.491716:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971614.491719:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971614.491727:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793b80 x1875872189548800/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971614.492242:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580.
00000400:00000200:3.0:1788971614.492250:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0
00000400:00000010:3.0:1788971614.492254:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0.
00000100:00000200:3.0:1788971614.492263:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792300 x1875872189549056/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971614.492273:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792300 x1875872189549056/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971614.492310:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126e70.
00000400:00000200:3.0:1788971614.492542:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b770
00000400:00000010:3.0:1788971614.492545:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b770.
00000100:00000200:3.0:1788971614.492553:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793800 x1875872189548928/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971614.492566:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793800 x1875872189548928/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971614.492590:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343000.
02000000:00000010:3.0:1788971614.492597:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00.
00000100:00000010:3.0:1788971614.492603:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971614.492607:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343e00.
02000000:00000010:3.0:1788971614.492624:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:3.0:1788971614.492626:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800.
00000400:00000010:3.0:1788971614.493389:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126318.
00000400:00000200:3.0:1788971614.493407:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18
00000400:00000010:3.0:1788971614.493410:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18.
00000100:00000200:3.0:1788971614.493418:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793b80 x1875872189548800/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971614.493428:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793b80 x1875872189548800/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971630 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971614.493447:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1342c00.
02000000:00000010:3.0:1788971614.493451:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600.
00000100:00000010:3.0:1788971614.493454:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80.
00000100:00000010:3.0:1788971615.506005:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726ca80.
00000100:00000010:2.0:1788971615.507152:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c840.
00000100:00000010:2.0:1788971619.602375:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00.
02000000:00000010:2.0:1788971619.602473:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63a00.
00000100:00000010:2.0:1788971619.602514:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788971619.602520:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00.
00000100:00000010:2.0:1788971619.602528:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e300.
02000000:00000010:2.0:1788971619.602534:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63b00.
02000000:00000010:3.0:1788971619.602841:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002600.
00000400:00000200:3.0:1788971619.602862:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971619.602868:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68.
00000400:00000010:3.0:1788971619.602877:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788971619.602886:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549184, portal 25
00000400:00000010:3.0:1788971619.602893:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873c38.
00000100:00000200:3.0:1788971619.602896:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189549184, offset 0
00000400:00000200:3.0:1788971619.602907:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971619.602926:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.602937:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971619.602943:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971619.602947:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971619.602951:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971619.602954:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971619.602958:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [282:282:256:4294967295] 192.168.203.150@tcp LPNI seq info [282:282:8:4294967295]
00000400:00000200:3.0:1788971619.602963:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.602973:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971619.602980:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971619.603002:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600.
00000800:00000200:3.0:1788971619.603010:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971619.603019:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971619.603025:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971619.603048:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788971619.603054:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391600.
00000100:00000010:3.0:1788971619.603067:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391240.
02000000:00000010:3.0:1788971619.603079:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003800.
00000400:00000200:3.0:1788971619.603083:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971619.603086:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210.
00000400:00000010:3.0:1788971619.603090:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788971619.603093:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549440, portal 10
00000400:00000010:3.0:1788971619.603098:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873bb0.
00000100:00000200:3.0:1788971619.603100:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189549440, offset 0
00000400:00000200:3.0:1788971619.603105:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971619.603108:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.603112:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971619.603116:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971619.603119:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971619.603122:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971619.603126:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971619.603129:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [283:283:255:4294967295] 192.168.203.150@tcp LPNI seq info [283:283:7:4294967295]
00000400:00000200:3.0:1788971619.603135:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.603140:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971619.603145:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971619.603149:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67200.
00000800:00000200:3.0:1788971619.603153:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971619.603157:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971619.603160:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971619.603166:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971619.603181:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971619.603186:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600.
00000400:00000200:2.0:1788971619.603196:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
02000000:00000010:3.0:1788971619.603198:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002800.
00000400:00000200:2.0:1788971619.603205:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971619.603206:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971619.603213:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10.
00000400:00000010:3.0:1788971619.603217:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0.
00000100:00000200:3.0:1788971619.603220:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549312, portal 10
00000400:00000200:2.0:1788971619.603225:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873c38
00000400:00000010:3.0:1788971619.603226:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873110.
00000400:00000010:2.0:1788971619.603231:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873c38.
00000100:00000200:2.0:1788971619.603245:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840dc00 x1875872189549184/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971619.603339:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971619.603341:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67200.
00000400:00000200:2.0:1788971619.603345:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971619.603349:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971619.604139:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189549312, offset 0
00000400:00000200:3.0:1788971619.604153:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971619.604161:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.604166:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971619.604170:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971619.604173:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971619.604176:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971619.604179:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971619.604183:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [284:284:255:4294967295] 192.168.203.150@tcp LPNI seq info [284:284:7:4294967295]
00000400:00000200:3.0:1788971619.604187:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.604194:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971619.604199:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971619.604204:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00.
00000800:00000200:3.0:1788971619.604210:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971619.604215:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971619.604219:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971619.604227:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605526:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605535:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971619.605543:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605547:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605656:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605662:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808a80
00000400:00000200:3.0:1788971619.605667:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605674:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xe7d [1] + 224
00000400:00000200:3.0:1788971619.605765:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971619.605772:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971619.605789:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840dc00 x1875872189549184/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971635 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971619.605910:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605914:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971619.605977:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605980:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605983:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605986:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808b80
00000400:00000200:3.0:1788971619.605989:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.605994:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe8d [1] + 224
00000400:00000200:3.0:1788971619.606004:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971619.606007:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971619.606013:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840e300 x1875872189549440/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971619.606404:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68.
00000400:00000200:3.0:1788971619.606409:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908
00000400:00000010:3.0:1788971619.606412:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788971619.606419:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840dc00 x1875872189549184/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971635 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971619.606427:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840dc00 x1875872189549184/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971635 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971619.606454:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210.
00000400:00000200:3.0:1788971619.606457:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873660
00000400:00000010:3.0:1788971619.606458:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788971619.606461:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840e300 x1875872189549440/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971619.606468:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840e300 x1875872189549440/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 2 fl Rpc:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971619.606489:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002600.
02000000:00000010:3.0:1788971619.606495:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63a00.
00000100:00000010:3.0:1788971619.606501:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00.
00000400:00000200:2.0:1788971619.610049:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873bb0
00000400:00000010:2.0:1788971619.610084:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873bb0.
00000100:00000200:2.0:1788971619.610110:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840e300 x1875872189549440/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 2 fl UnregRPC:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971619.610280:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971619.610285:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00.
00000400:00000200:2.0:1788971619.610291:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971619.610297:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971619.610302:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873110
00000400:00000010:2.0:1788971619.610304:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873110.
00000100:00000200:2.0:1788971619.610310:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ed80 x1875872189549312/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971619.610392:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003800.
02000000:00000010:2.0:1788971619.610401:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63b00.
00000100:00000010:2.0:1788971619.610415:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e300.
00000400:00000200:3.0:1788971619.612966:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.612978:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971619.612989:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.612994:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.613001:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.613006:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808b00
00000400:00000200:3.0:1788971619.613012:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971619.613019:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xe9d [1] + 224
00000400:00000200:3.0:1788971619.613035:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971619.613040:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971619.613056:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ed80 x1875872189549312/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971619.614909:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10.
00000400:00000200:2.0:1788971619.614926:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0
00000400:00000010:2.0:1788971619.614929:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0.
00000100:00000200:2.0:1788971619.614945:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ed80 x1875872189549312/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971619.614958:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ed80 x1875872189549312/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971635 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971619.615000:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002800.
02000000:00000010:2.0:1788971619.615008:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00.
00000100:00000010:2.0:1788971619.615019:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80.
00000100:00000010:2.0:1788971620.626287:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391600.
00000100:00000010:3.0:1788971620.628375:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391240.
00000100:00000010:3.0:1788971624.724983:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80.
02000000:00000010:3.0:1788971624.725015:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300.
00000100:00000010:3.0:1788971624.725045:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800.
02000000:00000010:3.0:1788971624.725052:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00.
00000100:00000010:3.0:1788971624.725061:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300.
02000000:00000010:3.0:1788971624.725066:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
02000000:00000010:2.0:1788971624.725446:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1342c00.
00000400:00000200:2.0:1788971624.725457:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971624.725462:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f78.
00000400:00000010:2.0:1788971624.725469:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788971624.725476:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549568, portal 25
00000400:00000010:2.0:1788971624.725481:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971624.725500:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189549568, offset 0
00000400:00000200:2.0:1788971624.725514:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971624.725523:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971624.725533:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971624.725543:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971624.725548:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971624.725553:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971624.725557:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971624.725562:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [285:285:256:4294967295] 192.168.203.150@tcp LPNI seq info [285:285:8:4294967295]
00000400:00000200:2.0:1788971624.725566:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971624.725574:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971624.725580:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971624.725585:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00.
00000800:00000200:2.0:1788971624.725592:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971624.725599:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971624.725603:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971624.725619:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971624.725623:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c840.
00000100:00000010:2.0:1788971624.725635:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c060.
02000000:00000010:2.0:1788971624.725648:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343e00.
00000400:00000200:2.0:1788971624.725653:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971624.725656:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0.
00000400:00000010:2.0:1788971624.725662:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b770.
00000100:00000200:2.0:1788971624.725665:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549824, portal 10
00000400:00000010:2.0:1788971624.725671:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971624.725673:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189549824, offset 0
00000400:00000200:2.0:1788971624.725678:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971624.725683:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971624.725688:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971624.725706:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971624.725709:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971624.725720:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971624.725722:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971624.725725:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [286:286:255:4294967295] 192.168.203.150@tcp LPNI seq info [286:286:7:4294967295]
00000400:00000200:2.0:1788971624.725730:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971624.725736:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971624.725741:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971624.725745:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63b00.
00000800:00000200:2.0:1788971624.725750:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971624.725755:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971624.725758:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971624.725766:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788971624.725785:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343000.
00000400:00000200:2.0:1788971624.725790:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971624.725793:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160.
00000400:00000010:2.0:1788971624.725797:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971624.725800:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549696, portal 10
00000400:00000010:2.0:1788971624.725805:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971624.725808:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189549696, offset 0
00000400:00000200:2.0:1788971624.725840:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971624.725845:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971624.725848:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971624.725851:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971624.725853:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971624.725856:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971624.725858:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971624.725860:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [287:287:254:4294967295] 192.168.203.150@tcp LPNI seq info [287:287:6:4294967295]
00000400:00000200:2.0:1788971624.725865:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971624.725871:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971624.725876:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971624.725891:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63000.
00000800:00000200:2.0:1788971624.725902:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971624.725906:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971624.725909:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971624.725915:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971624.725950:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971624.725954:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00.
00000400:00000200:2.0:1788971624.725958:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971624.725963:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971624.725968:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18
00000400:00000010:2.0:1788971624.725970:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971624.725978:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793b80 x1875872189549568/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971624.726001:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971624.726003:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63b00.
00000400:00000200:2.0:1788971624.726006:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971624.726009:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971624.726012:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0
00000400:00000010:2.0:1788971624.726014:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971624.726018:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792300 x1875872189549824/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971624.726057:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971624.726060:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63000.
00000400:00000200:2.0:1788971624.726063:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971624.726066:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971624.726069:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198
00000400:00000010:2.0:1788971624.726070:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971624.726075:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793800 x1875872189549696/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971624.730722:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730735:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971624.730745:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730750:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730756:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730761:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808c80
00000400:00000200:3.0:1788971624.730767:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730792:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xecd [1] + 224
00000400:00000200:3.0:1788971624.730805:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971624.730811:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971624.730866:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793800 x1875872189549696/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971624.730901:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730905:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971624.730908:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730911:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730913:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730916:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808c00
00000400:00000200:3.0:1788971624.730920:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730924:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xead [1] + 224
00000400:00000200:3.0:1788971624.730930:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971624.730934:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971624.730941:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793b80 x1875872189549568/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971624.730956:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730960:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971624.730964:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730967:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730970:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730972:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808d00
00000400:00000200:3.0:1788971624.730976:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971624.730979:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xebd [1] + 224
00000400:00000200:3.0:1788971624.730990:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971624.730993:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971624.730999:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792300 x1875872189549824/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971624.733543:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160.
00000400:00000200:2.0:1788971624.733555:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:2.0:1788971624.733557:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971624.733571:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793800 x1875872189549696/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971624.733599:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793800 x1875872189549696/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971624.733631:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f78.
00000400:00000200:2.0:1788971624.733635:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0
00000400:00000010:2.0:1788971624.733637:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788971624.733642:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793b80 x1875872189549568/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971624.733652:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793b80 x1875872189549568/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971624.733664:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0.
00000400:00000200:2.0:1788971624.733666:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b770
00000400:00000010:2.0:1788971624.733668:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b770.
00000100:00000200:2.0:1788971624.733672:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792300 x1875872189549824/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971624.733681:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792300 x1875872189549824/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971640 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971624.733709:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343000.
02000000:00000010:2.0:1788971624.733716:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00.
00000100:00000010:2.0:1788971624.733721:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800.
02000000:00000010:2.0:1788971624.733725:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1342c00.
02000000:00000010:2.0:1788971624.733727:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300.
00000100:00000010:2.0:1788971624.733730:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80.
02000000:00000010:2.0:1788971624.733732:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343e00.
02000000:00000010:2.0:1788971624.733735:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:2.0:1788971624.733737:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300.
00000100:00000010:2.0:1788971625.746001:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c840.
00000100:00000010:3.0:1788971625.746265:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c060.
00000100:00000010:2.0:1788971629.842088:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788971629.842116:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63b00.
00000100:00000010:2.0:1788971629.842196:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e300.
02000000:00000010:2.0:1788971629.842203:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00.
00000100:00000010:2.0:1788971629.842209:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100.
02000000:00000010:2.0:1788971629.842213:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63800.
02000000:00000010:2.0:1788971629.842251:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343e00.
00000400:00000200:2.0:1788971629.842262:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971629.842266:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0.
00000400:00000010:2.0:1788971629.842275:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b770.
00000100:00000200:2.0:1788971629.842284:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550080, portal 10
00000400:00000010:2.0:1788971629.842302:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788971629.842305:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189550080, offset 0
00000400:00000200:2.0:1788971629.842316:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971629.842327:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971629.842337:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971629.842343:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971629.842348:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971629.842353:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971629.842356:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971629.842360:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [288:288:256:4294967295] 192.168.203.150@tcp LPNI seq info [288:288:8:4294967295]
00000400:00000200:2.0:1788971629.842365:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788971629.842557:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0ff2600.
00000400:00000200:3.0:1788971629.842564:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971629.842575:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210.
00000400:00000010:3.0:1788971629.842581:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788971629.842584:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189549952, portal 25
00000400:00000010:3.0:1788971629.842588:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788971629.842590:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189549952, offset 0
00000400:00000200:3.0:1788971629.842596:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971629.842624:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971629.842631:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971629.842638:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63a00.
00000800:00000200:2.0:1788971629.842645:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971629.842653:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971629.842659:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971629.842677:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971629.842682:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c840.
00000100:00000010:2.0:1788971629.842696:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c726c1e0.
00000800:00000200:2.0:1788971629.843118:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971629.843124:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63a00.
00000400:00000200:2.0:1788971629.843149:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971629.843156:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971629.850455:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.850475:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971629.850483:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971629.850487:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971629.850492:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971629.850496:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971629.850500:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [289:289:255:4294967295] 192.168.203.150@tcp LPNI seq info [289:289:7:4294967295]
00000400:00000200:3.0:1788971629.850505:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.850516:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971629.850525:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971629.850534:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67200.
00000800:00000200:3.0:1788971629.850543:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971629.850551:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971629.850560:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971629.850574:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788971629.850618:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0ff3e00.
00000400:00000200:3.0:1788971629.850623:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971629.850627:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68.
00000400:00000010:3.0:1788971629.850635:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873d48.
00000100:00000200:3.0:1788971629.850644:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550208, portal 10
00000400:00000010:3.0:1788971629.850653:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873a18.
00000100:00000200:3.0:1788971629.850674:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189550208, offset 0
00000400:00000200:3.0:1788971629.850682:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971629.850690:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.850694:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971629.850696:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971629.850698:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971629.850701:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971629.851080:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971629.851084:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [290:290:254:4294967295] 192.168.203.150@tcp LPNI seq info [290:290:6:4294967295]
00000400:00000200:3.0:1788971629.851088:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851095:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971629.851099:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971629.851105:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300.
00000800:00000200:3.0:1788971629.851109:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971629.851114:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971629.851118:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971629.851125:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851183:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851187:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971629.851193:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851197:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851201:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851204:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808e00
00000400:00000200:3.0:1788971629.851209:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.851215:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xedd [1] + 224
00000400:00000200:3.0:1788971629.851240:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971629.851246:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971629.851262:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840e300 x1875872189550080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971629.851330:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0.
00000400:00000200:3.0:1788971629.851341:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b770
00000400:00000010:3.0:1788971629.851344:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b770.
00000100:00000200:3.0:1788971629.851351:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840e300 x1875872189550080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971629.851362:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840e300 x1875872189550080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 2 fl Rpc:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971629.852398:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0
00000400:00000010:2.0:1788971629.852405:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788971629.852419:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840e300 x1875872189550080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 2 fl UnregRPC:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971629.852501:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971629.852505:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67200.
00000400:00000200:2.0:1788971629.852511:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971629.852515:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971629.852518:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908
00000400:00000010:2.0:1788971629.852520:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908.
00000100:00000200:2.0:1788971629.852527:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ed80 x1875872189549952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971629.852606:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971629.852609:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971629.852612:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971629.852616:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971629.852619:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873a18
00000400:00000010:2.0:1788971629.852621:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873a18.
00000100:00000200:2.0:1788971629.852626:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f100 x1875872189550208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971629.853170:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343e00.
02000000:00000010:3.0:1788971629.853179:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00.
00000100:00000010:3.0:1788971629.853184:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e300.
00000400:00000200:3.0:1788971629.859239:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859252:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971629.859263:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859268:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859275:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859279:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808d80
00000400:00000200:3.0:1788971629.859297:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859309:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xeed [1] + 224
00000400:00000200:3.0:1788971629.859324:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971629.859333:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971629.859351:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ed80 x1875872189549952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971645 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971629.859400:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859404:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971629.859408:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859410:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859414:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859417:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808e80
00000400:00000200:3.0:1788971629.859420:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971629.859424:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xefd [1] + 224
00000400:00000200:3.0:1788971629.859433:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971629.859457:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971629.859465:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f100 x1875872189550208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971629.862428:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210.
00000400:00000200:2.0:1788971629.862447:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873660
00000400:00000010:2.0:1788971629.862450:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873660.
00000100:00000200:2.0:1788971629.862468:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ed80 x1875872189549952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971645 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971629.862483:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ed80 x1875872189549952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971645 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971629.862514:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68.
00000400:00000200:2.0:1788971629.862518:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873d48
00000400:00000010:2.0:1788971629.862540:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873d48.
00000100:00000200:2.0:1788971629.862547:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f100 x1875872189550208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971629.862556:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f100 x1875872189550208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971645 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971629.862580:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0ff2600.
02000000:00000010:2.0:1788971629.862589:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63b00.
00000100:00000010:2.0:1788971629.862595:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788971629.862599:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0ff3e00.
02000000:00000010:2.0:1788971629.862601:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63800.
00000100:00000010:2.0:1788971629.862603:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100.
00000100:00000010:2.0:1788971630.866486:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c840.
00000100:00000010:3.0:1788971630.868050:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c726c1e0.
00000100:00000010:0.0:1788971634.967439:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800.
02000000:00000010:0.0:1788971634.967473:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100.
00000100:00000010:0.0:1788971634.967529:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500.
02000000:00000010:0.0:1788971634.967535:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b000.
00000100:00000010:0.0:1788971634.967543:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700.
02000000:00000010:0.0:1788971634.967547:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bd00.
02000000:00000010:0.0:1788971634.967588:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9ec00.
00000400:00000200:0.0:1788971634.967599:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971634.967605:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971634.967623:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971634.967633:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550464, portal 10
00000400:00000010:0.0:1788971634.967642:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8f68.
00000100:00000200:0.0:1788971634.967646:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189550464, offset 0
00000400:00000200:0.0:1788971634.967658:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971634.967670:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971634.967679:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971634.967685:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971634.967690:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971634.967694:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971634.967697:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971634.967701:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [291:291:256:4294967295] 192.168.203.150@tcp LPNI seq info [291:291:8:4294967295]
00000400:00000200:0.0:1788971634.967707:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971634.967718:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971634.967725:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971634.967733:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00.
00000800:00000200:0.0:1788971634.967742:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971634.967751:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971634.967756:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1
02000000:00000010:1.0:1788971634.967843:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46ca800.
00000400:00000200:1.0:1788971634.967851:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971634.967855:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000010:1.0:1788971634.967860:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971634.967863:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550336, portal 25
00000400:00000010:1.0:1788971634.967867:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525330.
00000100:00000200:1.0:1788971634.967869:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189550336, offset 0
00000400:00000200:1.0:1788971634.967876:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971634.967880:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971634.967884:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971634.967887:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971634.967890:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971634.967905:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971634.967909:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971634.967913:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [292:292:255:4294967295] 192.168.203.150@tcp LPNI seq info [292:292:7:4294967295]
00000400:00000200:1.0:1788971634.967917:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971634.967924:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971634.967930:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971634.967935:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f100.
00000800:00000200:1.0:1788971634.967939:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971634.967944:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971634.967948:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971634.967953:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971634.967957:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971634.967961:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c51e0.
00000100:00000010:0.0:1788971634.967976:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c53c0.
02000000:00000010:1.0:1788971634.971945:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46cb200.
00000400:00000200:1.0:1788971634.971964:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971634.971970:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000010:1.0:1788971634.971979:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525b28.
00000100:00000200:1.0:1788971634.971987:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550592, portal 10
00000400:00000010:1.0:1788971634.971996:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8.
00000100:00000200:1.0:1788971634.972000:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189550592, offset 0
00000400:00000200:1.0:1788971634.972012:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971634.972023:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971634.972034:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971634.972040:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971634.972045:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971634.972049:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971634.972052:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971634.972057:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [293:293:254:4294967295] 192.168.203.150@tcp LPNI seq info [293:293:6:4294967295]
00000400:00000200:1.0:1788971634.972062:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971634.972074:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971634.972080:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971634.972088:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0ff00.
00000800:00000200:1.0:1788971634.972118:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971634.972127:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971634.972132:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0ff00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971634.972141:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971634.975942:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971634.975954:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00.
00000400:00000200:2.0:1788971634.975965:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971634.975977:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971634.975986:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8f68
00000400:00000010:2.0:1788971634.975990:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8f68.
00000100:00000200:2.0:1788971634.976007:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a89500 x1875872189550464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971634.976045:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971634.976047:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f100.
00000400:00000200:2.0:1788971634.976051:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971634.976055:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971634.976057:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525330
00000400:00000010:2.0:1788971634.976059:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525330.
00000100:00000200:2.0:1788971634.976066:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a8b800 x1875872189550336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971634.976154:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971634.976157:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0ff00.
00000400:00000200:2.0:1788971634.976160:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971634.976177:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971634.976180:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8
00000400:00000010:2.0:1788971634.976183:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8.
00000100:00000200:2.0:1788971634.976190:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88700 x1875872189550592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971634.979090:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971634.979104:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971634.979113:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971634.979119:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971634.979125:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971634.979130:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887808f00
00000400:00000200:3.0:1788971634.979136:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971634.979144:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xf15 [1] + 224
00000400:00000200:3.0:1788971634.979160:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971634.979165:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971634.979182:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a8b800 x1875872189550336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971634.980034:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000200:1.0:1788971634.980044:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220
00000400:00000010:1.0:1788971634.980047:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971634.980059:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a8b800 x1875872189550336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971634.980069:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a8b800 x1875872189550336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971634.980104:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46ca800.
02000000:00000010:1.0:1788971634.980113:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100.
00000100:00000010:1.0:1788971634.980121:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800.
00000400:00000200:2.0:1788971634.988758:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988782:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971634.988797:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988803:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988811:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988846:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887808f80
00000400:00000200:2.0:1788971634.988853:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988861:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf0d [1] + 224
00000400:00000200:2.0:1788971634.988877:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971634.988884:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971634.988902:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a89500 x1875872189550464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971634.988953:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988959:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971634.988964:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988968:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988972:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988976:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809000
00000400:00000200:2.0:1788971634.988980:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971634.988986:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf1d [1] + 224
00000400:00000200:2.0:1788971634.989051:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971634.989057:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971634.989066:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88700 x1875872189550592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971634.990467:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000200:1.0:1788971634.990480:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525b28
00000400:00000010:1.0:1788971634.990483:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525b28.
00000100:00000200:1.0:1788971634.990497:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88700 x1875872189550592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971634.990510:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88700 x1875872189550592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971634.990556:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46cb200.
02000000:00000010:1.0:1788971634.990565:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bd00.
00000100:00000010:1.0:1788971634.990571:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700.
00000400:00000010:0.0:1788971634.992017:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971634.992030:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971634.992034:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971634.992049:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a89500 x1875872189550464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971634.992062:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a89500 x1875872189550464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971650 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971634.992100:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9ec00.
02000000:00000010:0.0:1788971634.992107:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b000.
00000100:00000010:0.0:1788971634.992114:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500.
00000100:00000010:2.0:1788971635.986500:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c51e0.
00000100:00000010:3.0:1788971635.986752:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c53c0.
00000100:00000010:3.0:1788971640.083491:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793480.
02000000:00000010:3.0:1788971640.083530:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600.
02000000:00000010:2.0:1788971640.083626:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343e00.
00000400:00000200:2.0:1788971640.083637:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971640.083641:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f78.
00000400:00000010:2.0:1788971640.083648:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788971640.083656:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550720, portal 25
00000400:00000010:2.0:1788971640.083662:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971640.083664:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189550720, offset 0
00000400:00000200:2.0:1788971640.083673:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971640.083682:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.083692:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971640.083698:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971640.083704:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971640.083708:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971640.083711:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971640.083715:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [294:294:256:4294967295] 192.168.203.150@tcp LPNI seq info [294:294:8:4294967295]
00000400:00000200:2.0:1788971640.083720:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.083731:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971640.083737:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971640.083744:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00.
00000800:00000200:2.0:1788971640.083752:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971640.083762:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971640.083768:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971640.083788:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971640.083800:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e8a0.
00000100:00000010:2.0:1788971640.083850:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0.
00000100:00000010:3.0:1788971640.083914:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100.
02000000:00000010:3.0:1788971640.083922:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00.
02000000:00000010:2.0:1788971640.084026:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1285400.
00000400:00000200:2.0:1788971640.084031:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971640.084034:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160.
00000400:00000010:2.0:1788971640.084055:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971640.084058:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550848, portal 10
00000400:00000010:2.0:1788971640.084064:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971640.084066:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189550848, offset 0
00000400:00000200:2.0:1788971640.084070:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971640.084075:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.084078:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971640.084081:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971640.084083:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971640.084086:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971640.084088:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971640.084091:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [295:295:255:4294967295] 192.168.203.150@tcp LPNI seq info [295:295:7:4294967295]
00000400:00000200:2.0:1788971640.084096:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.084102:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971640.084106:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971640.084111:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c200.
00000800:00000200:2.0:1788971640.084115:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971640.084119:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971640.084122:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971640.084129:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971640.084208:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971640.084226:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00.
00000400:00000200:2.0:1788971640.084236:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971640.084241:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971640.084247:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:2.0:1788971640.084249:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971640.084260:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793480 x1875872189550720/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971640.084422:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971640.084425:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c200.
00000400:00000200:2.0:1788971640.084431:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971640.084440:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971640.084444:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0
00000400:00000010:2.0:1788971640.084445:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971640.084451:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793100 x1875872189550848/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788971640.090048:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791f80.
02000000:00000010:3.0:1788971640.090080:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300.
02000000:00000010:3.0:1788971640.090150:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b000.
00000400:00000200:3.0:1788971640.090161:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971640.090167:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18.
00000400:00000010:3.0:1788971640.090175:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0.
00000100:00000200:3.0:1788971640.090189:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189550976, portal 10
00000400:00000010:3.0:1788971640.090198:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220.
00000100:00000200:3.0:1788971640.090201:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189550976, offset 0
00000400:00000200:3.0:1788971640.090242:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971640.090253:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971640.090263:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971640.090269:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971640.090273:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971640.090278:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971640.090281:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971640.090284:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [296:296:256:4294967295] 192.168.203.150@tcp LPNI seq info [296:296:8:4294967295]
00000400:00000200:3.0:1788971640.090289:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971640.090300:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971640.090306:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971640.090312:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67200.
00000800:00000200:3.0:1788971640.090319:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971640.090327:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971640.090333:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971640.097320:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097332:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971640.097347:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097353:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097360:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097364:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809100
00000400:00000200:2.0:1788971640.097371:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097378:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf35 [1] + 224
00000400:00000200:2.0:1788971640.097393:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971640.097398:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971640.097417:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793100 x1875872189550848/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971640.097569:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971640.097574:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67200.
00000400:00000200:2.0:1788971640.097578:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971640.097582:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971640.097586:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220
00000400:00000010:2.0:1788971640.097588:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220.
00000100:00000200:2.0:1788971640.097596:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8791f80 x1875872189550976/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971640.097618:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097622:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971640.097626:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097629:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097632:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097652:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809080
00000400:00000200:2.0:1788971640.097657:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.097665:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xf25 [1] + 224
00000400:00000200:2.0:1788971640.097675:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971640.097678:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971640.097686:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793480 x1875872189550720/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971640.097757:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160.
00000400:00000200:2.0:1788971640.097762:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198
00000400:00000010:2.0:1788971640.097763:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971640.097769:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793100 x1875872189550848/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971640.097778:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793100 x1875872189550848/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971640.097848:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f78.
00000400:00000200:2.0:1788971640.097851:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0
00000400:00000010:2.0:1788971640.097853:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788971640.097858:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793480 x1875872189550720/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971640.097866:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793480 x1875872189550720/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971640.097882:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1285400.
02000000:00000010:2.0:1788971640.097889:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00.
00000100:00000010:2.0:1788971640.097894:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100.
02000000:00000010:2.0:1788971640.097898:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343e00.
02000000:00000010:2.0:1788971640.097902:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600.
00000100:00000010:2.0:1788971640.097904:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793480.
00000400:00000200:3.0:1788971640.098284:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.099208:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.099228:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971640.099250:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.099254:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.099258:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.099262:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809180
00000400:00000200:2.0:1788971640.099267:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971640.099274:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf45 [1] + 224
00000400:00000200:2.0:1788971640.099287:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971640.099292:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971640.099304:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8791f80 x1875872189550976/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971640.100887:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18.
00000400:00000200:3.0:1788971640.100901:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0
00000400:00000010:3.0:1788971640.100904:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0.
00000100:00000200:3.0:1788971640.100916:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8791f80 x1875872189550976/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971640.100928:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8791f80 x1875872189550976/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971656 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971640.100969:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b000.
02000000:00000010:3.0:1788971640.100980:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300.
00000100:00000010:3.0:1788971640.100985:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791f80.
00000100:00000010:2.0:1788971641.105990:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e8a0.
00000100:00000010:3.0:1788971641.105994:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0.
00000100:00000010:1.0:1788971645.204237:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788971645.204273:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f900.
00000100:00000010:1.0:1788971645.204311:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80.
02000000:00000010:1.0:1788971645.204317:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00.
00000100:00000010:1.0:1788971645.204323:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3b80.
02000000:00000010:1.0:1788971645.204327:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f000.
02000000:00000010:0.0:1788971645.204423:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc5e00.
00000400:00000200:0.0:1788971645.204437:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971645.204442:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971645.204451:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971645.204461:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551104, portal 25
00000400:00000010:0.0:1788971645.204467:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8330.
00000100:00000200:0.0:1788971645.204471:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189551104, offset 0
00000400:00000200:0.0:1788971645.204484:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971645.204495:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971645.204506:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971645.204512:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971645.204518:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971645.204522:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971645.204525:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971645.204529:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [297:297:256:4294967295] 192.168.203.150@tcp LPNI seq info [297:297:8:4294967295]
00000400:00000200:0.0:1788971645.204534:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971645.204546:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971645.204553:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971645.204560:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000.
00000800:00000200:0.0:1788971645.204568:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971645.204595:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971645.204601:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971645.204624:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971645.204630:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5540.
00000100:00000010:0.0:1788971645.204643:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c50c0.
02000000:00000010:0.0:1788971645.204659:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4e00.
00000400:00000200:0.0:1788971645.204663:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971645.204666:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971645.204670:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a85d8.
00000100:00000200:0.0:1788971645.204674:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551360, portal 10
00000400:00000010:0.0:1788971645.204679:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550.
00000100:00000200:0.0:1788971645.204681:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189551360, offset 0
00000400:00000200:0.0:1788971645.204686:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971645.204690:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971645.204693:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971645.204696:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971645.204699:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971645.204701:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971645.204703:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971645.204706:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [298:298:255:4294967295] 192.168.203.150@tcp LPNI seq info [298:298:7:4294967295]
00000400:00000200:0.0:1788971645.204710:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971645.204715:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971645.204720:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971645.204724:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f900.
00000800:00000200:0.0:1788971645.204727:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971645.204731:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971645.204734:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971645.204739:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971645.204759:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4472800.
00000400:00000200:0.0:1788971645.204764:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971645.204771:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971645.204775:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088.
00000100:00000200:0.0:1788971645.204778:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551232, portal 10
00000400:00000010:0.0:1788971645.204782:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8.
00000100:00000200:0.0:1788971645.204785:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189551232, offset 0
00000400:00000200:0.0:1788971645.204789:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971645.204793:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971645.204796:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971645.204799:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971645.204802:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971645.204804:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971645.204806:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971645.204809:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [299:299:254:4294967295] 192.168.203.150@tcp LPNI seq info [299:299:6:4294967295]
00000400:00000200:0.0:1788971645.204832:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971645.204838:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971645.204843:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971645.204846:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f500.
00000800:00000200:0.0:1788971645.204850:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971645.204854:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971645.204856:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971645.204862:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971645.205903:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971645.205914:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000.
00000400:00000200:3.0:1788971645.205925:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971645.205944:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971645.205952:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8330
00000400:00000010:3.0:1788971645.205954:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8330.
00000100:00000200:3.0:1788971645.205967:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189551104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971645.206002:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971645.206004:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f900.
00000400:00000200:3.0:1788971645.206007:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971645.206010:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971645.206013:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550
00000400:00000010:3.0:1788971645.206014:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550.
00000100:00000200:3.0:1788971645.206020:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3b80 x1875872189551360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971645.206108:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788971645.206111:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f500.
00000400:00000200:3.0:1788971645.206114:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971645.206118:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971645.206120:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8
00000400:00000010:3.0:1788971645.206122:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8.
00000100:00000200:3.0:1788971645.206127:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f1f80 x1875872189551232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971645.208581:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208592:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971645.208602:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208608:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208614:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208619:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809200
00000400:00000200:2.0:1788971645.208626:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208635:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xf55 [1] + 224
00000400:00000200:2.0:1788971645.208651:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971645.208655:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971645.208667:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189551104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971645.208757:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208760:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971645.208763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208764:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208766:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208768:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809300
00000400:00000200:2.0:1788971645.208771:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208791:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf5d [1] + 224
00000400:00000200:2.0:1788971645.208796:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971645.208799:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971645.208804:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3b80 x1875872189551360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971645.208849:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208851:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971645.208855:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208856:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208858:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208860:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809280
00000400:00000200:2.0:1788971645.208862:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971645.208865:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf65 [1] + 224
00000400:00000200:2.0:1788971645.208872:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971645.208874:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971645.208877:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f1f80 x1875872189551232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971645.209347:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971645.209354:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971645.209357:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971645.209365:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189551104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971645.209375:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189551104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971645.209512:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971645.209587:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a85d8
00000400:00000010:0.0:1788971645.209590:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a85d8.
00000100:00000200:0.0:1788971645.209609:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3b80 x1875872189551360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971645.209617:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3b80 x1875872189551360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971645.209645:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc5e00.
02000000:00000010:0.0:1788971645.209656:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f900.
00000100:00000010:0.0:1788971645.209662:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
02000000:00000010:0.0:1788971645.209666:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4e00.
02000000:00000010:0.0:1788971645.209669:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f000.
00000100:00000010:0.0:1788971645.209671:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3b80.
00000400:00000010:0.0:1788971645.209679:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971645.209681:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088
00000400:00000010:0.0:1788971645.209683:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088.
00000100:00000200:0.0:1788971645.209688:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f1f80 x1875872189551232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971645.209697:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f1f80 x1875872189551232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971661 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971645.209712:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4472800.
02000000:00000010:0.0:1788971645.209716:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00.
00000100:00000010:0.0:1788971645.209719:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80.
00000100:00000010:3.0:1788971646.226004:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c50c0.
00000100:00000010:2.0:1788971646.226007:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5540.
00000100:00000010:1.0:1788971650.322715:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700.
02000000:00000010:1.0:1788971650.322760:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f600.
00000100:00000010:1.0:1788971650.322796:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80.
02000000:00000010:1.0:1788971650.322801:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f800.
00000100:00000010:1.0:1788971650.322807:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100.
02000000:00000010:1.0:1788971650.322829:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00.
02000000:00000010:0.0:1788971650.322967:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4472800.
00000400:00000200:0.0:1788971650.322982:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971650.322987:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971650.322995:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088.
00000100:00000200:0.0:1788971650.323003:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551488, portal 25
00000400:00000010:0.0:1788971650.323010:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a85d8.
00000100:00000200:0.0:1788971650.323014:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189551488, offset 0
00000400:00000200:0.0:1788971650.323025:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971650.323034:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971650.323045:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971650.323071:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971650.323077:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971650.323082:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971650.323085:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971650.323090:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [300:300:256:4294967295] 192.168.203.150@tcp LPNI seq info [300:300:8:4294967295]
00000400:00000200:0.0:1788971650.323095:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971650.323107:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971650.323113:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971650.323120:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f100.
00000800:00000200:0.0:1788971650.323129:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971650.323138:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971650.323153:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971650.323180:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971650.323186:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5420.
00000100:00000010:0.0:1788971650.323203:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5a80.
02000000:00000010:0.0:1788971650.323219:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4e00.
00000400:00000200:0.0:1788971650.323223:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971650.323226:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971650.323230:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971650.323233:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551744, portal 10
00000400:00000010:0.0:1788971650.323245:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a87f8.
00000100:00000200:0.0:1788971650.323248:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189551744, offset 0
00000400:00000200:0.0:1788971650.323253:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971650.323258:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971650.323261:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971650.323264:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971650.323267:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971650.323269:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971650.323271:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971650.323274:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [301:301:255:4294967295] 192.168.203.150@tcp LPNI seq info [301:301:7:4294967295]
00000400:00000200:0.0:1788971650.323278:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971650.323291:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971650.323296:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971650.323300:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f400.
00000800:00000200:0.0:1788971650.323303:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971650.323308:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971650.323311:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971650.323317:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971650.323338:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc5e00.
00000400:00000200:0.0:1788971650.323342:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971650.323344:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971650.323348:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8bb0.
00000100:00000200:0.0:1788971650.323350:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551616, portal 10
00000400:00000010:0.0:1788971650.323355:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28.
00000100:00000200:0.0:1788971650.323357:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189551616, offset 0
00000400:00000200:0.0:1788971650.323361:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971650.323364:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971650.323368:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971650.323371:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971650.323373:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971650.323375:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971650.323377:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971650.323380:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [302:302:254:4294967295] 192.168.203.150@tcp LPNI seq info [302:302:6:4294967295]
00000400:00000200:0.0:1788971650.323385:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971650.323390:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971650.323394:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971650.323397:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00.
00000800:00000200:0.0:1788971650.323401:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971650.323405:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971650.323407:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971650.323411:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971650.323689:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971650.323695:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f100.
00000400:00000200:2.0:1788971650.323703:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971650.323710:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971650.323718:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a85d8
00000400:00000010:2.0:1788971650.323721:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a85d8.
00000100:00000200:2.0:1788971650.324385:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f0700 x1875872189551488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971650.324430:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971650.324433:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f400.
00000400:00000200:2.0:1788971650.324438:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971650.324441:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971650.324460:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a87f8
00000400:00000010:2.0:1788971650.324463:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a87f8.
00000100:00000200:2.0:1788971650.324470:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3100 x1875872189551744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971650.324851:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971650.324855:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00.
00000400:00000200:2.0:1788971650.324859:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971650.324864:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971650.324867:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28
00000400:00000010:2.0:1788971650.324869:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28.
00000100:00000200:2.0:1788971650.324876:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f2d80 x1875872189551616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971650.337800:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337812:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971650.337851:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337857:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337863:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337868:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809480
00000400:00000200:2.0:1788971650.337874:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337881:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf75 [1] + 224
00000400:00000200:2.0:1788971650.337894:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971650.337899:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971650.337918:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3100 x1875872189551744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971650.337962:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337966:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971650.337970:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337972:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337975:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.337978:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809400
00000400:00000200:2.0:1788971650.338000:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.338005:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf7d [1] + 224
00000400:00000200:2.0:1788971650.338013:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971650.338016:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971650.338024:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f2d80 x1875872189551616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971650.338039:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.338042:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971650.338047:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.338049:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.338052:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.338055:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809380
00000400:00000200:2.0:1788971650.338059:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971650.338063:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xf6d [1] + 224
00000400:00000200:2.0:1788971650.338085:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971650.338089:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971650.338096:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f0700 x1875872189551488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971650.338258:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971650.338265:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8bb0
00000400:00000010:0.0:1788971650.338268:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8bb0.
00000100:00000200:0.0:1788971650.338276:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f2d80 x1875872189551616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971650.338284:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f2d80 x1875872189551616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971650.338324:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971650.338327:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088
00000400:00000010:0.0:1788971650.338329:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088.
00000100:00000200:0.0:1788971650.338334:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f0700 x1875872189551488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971650.338343:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f0700 x1875872189551488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971650.338354:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971650.338356:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971650.338357:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971650.338361:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3100 x1875872189551744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971650.338370:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3100 x1875872189551744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971666 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971650.338388:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc5e00.
02000000:00000010:0.0:1788971650.338397:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f800.
00000100:00000010:0.0:1788971650.338403:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80.
02000000:00000010:0.0:1788971650.338408:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4472800.
02000000:00000010:0.0:1788971650.338412:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f600.
00000100:00000010:0.0:1788971650.338415:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700.
02000000:00000010:0.0:1788971650.338418:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4e00.
02000000:00000010:0.0:1788971650.338420:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00.
00000100:00000010:0.0:1788971650.338423:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100.
00000100:00000010:3.0:1788971651.346003:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5a80.
00000100:00000010:2.0:1788971651.346345:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5420.
00000100:00000010:2.0:1788971655.444904:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100.
02000000:00000010:2.0:1788971655.444937:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c200.
00000100:00000010:2.0:1788971655.444979:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788971655.444984:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4cd00.
00000100:00000010:2.0:1788971655.444992:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ce00.
02000000:00000010:2.0:1788971655.444996:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c600.
02000000:00000010:3.0:1788971655.445028:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800.
02000000:00000010:2.0:1788971655.445028:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1285400.
00000400:00000200:3.0:1788971655.445047:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.445048:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971655.445058:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18.
00000400:00000010:2.0:1788971655.445058:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f78.
00000400:00000010:3.0:1788971655.445073:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0.
00000400:00000010:2.0:1788971655.445073:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0.
00000100:00000200:3.0:1788971655.445085:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189551872, portal 25
00000400:00000010:3.0:1788971655.445093:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770.
00000100:00000200:3.0:1788971655.445099:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189551872, offset 0
00000100:00000200:2.0:1788971655.445101:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552000, portal 10
00000400:00000010:2.0:1788971655.445119:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198.
00000400:00000200:3.0:1788971655.445124:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000100:00000200:2.0:1788971655.445138:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189552000, offset 0
00000400:00000200:3.0:1788971655.445145:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.445150:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971655.445160:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971655.445166:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971655.445172:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971655.445176:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971655.445179:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971655.445183:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [303:303:256:4294967295] 192.168.203.150@tcp LPNI seq info [303:303:8:4294967295]
00000400:00000200:3.0:1788971655.445188:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971655.445197:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788971655.445207:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971655.445208:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971655.445217:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971655.445217:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971655.445224:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971655.445229:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000800:00000200:3.0:1788971655.445233:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000400:00000200:2.0:1788971655.445235:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971655.445240:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971655.445246:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [304:304:255:4294967295] 192.168.203.150@tcp LPNI seq info [304:304:7:4294967295]
00000800:00000200:3.0:1788971655.445247:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:2.0:1788971655.445254:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971655.445255:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971655.445263:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971655.445268:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971655.445272:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c100.
00000400:00000200:3.0:1788971655.445276:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971655.445281:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000100:00000010:3.0:1788971655.445285:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c6534420.
00000800:00000200:2.0:1788971655.445288:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971655.445291:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971655.445298:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971655.445352:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971655.445357:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971655.445366:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971655.445372:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971655.445377:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770
00000400:00000010:2.0:1788971655.445380:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770.
00000100:00000200:2.0:1788971655.445392:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f100 x1875872189551872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000010:3.0:1788971655.445465:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c6534b40.
02000000:00000010:3.0:1788971655.445498:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b200.
00000400:00000200:3.0:1788971655.445503:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971655.445505:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160.
00000400:00000010:3.0:1788971655.445509:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8.
00000100:00000200:3.0:1788971655.445524:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552128, portal 10
00000400:00000010:3.0:1788971655.445529:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28.
00000100:00000200:3.0:1788971655.445531:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189552128, offset 0
00000400:00000200:3.0:1788971655.445536:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971655.445540:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971655.446015:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971655.446021:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c100.
00000400:00000200:2.0:1788971655.446029:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971655.446034:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971655.446373:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971655.446377:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971655.446381:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971655.446384:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971655.446387:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971655.446397:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [305:305:255:4294967295] 192.168.203.150@tcp LPNI seq info [305:305:7:4294967295]
00000400:00000200:3.0:1788971655.446403:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971655.446410:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971655.446417:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971655.446423:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300.
00000800:00000200:3.0:1788971655.446429:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971655.446435:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971655.446439:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971655.446447:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.446462:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198
00000400:00000010:2.0:1788971655.446466:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971655.446477:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ed80 x1875872189552000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971655.447036:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971655.447040:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971655.447045:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971655.447050:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971655.447054:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28
00000400:00000010:2.0:1788971655.447056:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28.
00000100:00000200:2.0:1788971655.447063:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ce00 x1875872189552128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971655.447916:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.447923:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971655.447931:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.447936:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.447941:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.447946:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809580
00000400:00000200:2.0:1788971655.447952:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.447958:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xf95 [1] + 224
00000400:00000200:2.0:1788971655.447974:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971655.447978:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971655.447988:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ed80 x1875872189552000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971655.448088:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f78.
00000400:00000200:3.0:1788971655.448095:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0
00000400:00000010:3.0:1788971655.448098:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0.
00000100:00000200:3.0:1788971655.448113:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ed80 x1875872189552000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971655.448124:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ed80 x1875872189552000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971655.448162:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1285400.
02000000:00000010:3.0:1788971655.448170:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4cd00.
00000100:00000010:3.0:1788971655.448177:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80.
00000400:00000200:2.0:1788971655.460979:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.460990:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971655.461072:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.461078:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.461083:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.461088:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809500
00000400:00000200:2.0:1788971655.461118:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.461125:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xf85 [1] + 224
00000400:00000200:2.0:1788971655.461152:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971655.461157:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971655.461170:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f100 x1875872189551872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971655.465647:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18.
00000400:00000200:3.0:1788971655.465660:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0
00000400:00000010:3.0:1788971655.465663:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0.
00000100:00000200:3.0:1788971655.465677:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f100 x1875872189551872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971655.465691:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f100 x1875872189551872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971655.465722:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800.
02000000:00000010:3.0:1788971655.465729:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c200.
00000100:00000010:3.0:1788971655.465736:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100.
00000400:00000200:2.0:1788971655.465865:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.465871:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971655.465878:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.465883:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.465888:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.465892:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809600
00000400:00000200:2.0:1788971655.465896:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971655.465902:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xfa5 [1] + 224
00000400:00000200:2.0:1788971655.465914:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971655.465919:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971655.465928:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ce00 x1875872189552128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971655.465998:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160.
00000400:00000200:3.0:1788971655.466072:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8
00000400:00000010:3.0:1788971655.466074:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8.
00000100:00000200:3.0:1788971655.466080:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ce00 x1875872189552128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971655.466087:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ce00 x1875872189552128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971671 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971655.466114:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b200.
02000000:00000010:3.0:1788971655.466119:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c600.
00000100:00000010:3.0:1788971655.466121:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ce00.
00000100:00000010:2.0:1788971656.466009:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c6534420.
00000100:00000010:3.0:1788971656.466139:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c6534b40.
00000100:00000010:0.0:1788971660.565074:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500.
02000000:00000010:0.0:1788971660.565097:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060ff00.
00000100:00000010:0.0:1788971660.565128:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80.
02000000:00000010:0.0:1788971660.565133:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f400.
00000100:00000010:0.0:1788971660.565139:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88e00.
02000000:00000010:0.0:1788971660.565142:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100.
02000000:00000010:0.0:1788971660.565172:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4e00.
00000400:00000200:0.0:1788971660.565181:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971660.565185:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971660.565191:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971660.565198:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552256, portal 25
00000400:00000010:0.0:1788971660.565204:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088.
00000100:00000200:0.0:1788971660.565207:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189552256, offset 0
00000400:00000200:0.0:1788971660.565215:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971660.565223:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971660.565232:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971660.565237:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971660.565240:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971660.565244:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971660.565247:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971660.565261:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [306:306:256:4294967295] 192.168.203.150@tcp LPNI seq info [306:306:8:4294967295]
00000400:00000200:0.0:1788971660.565265:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971660.565274:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971660.565280:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971660.565285:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f500.
00000800:00000200:0.0:1788971660.565291:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971660.565298:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971660.565303:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971660.565322:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971660.565328:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5cc0.
00000100:00000010:0.0:1788971660.565341:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d80.
02000000:00000010:0.0:1788971660.565376:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc5e00.
00000400:00000200:0.0:1788971660.565380:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971660.565383:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971660.565386:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8bb0.
00000100:00000200:0.0:1788971660.565389:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552512, portal 10
00000400:00000010:0.0:1788971660.565395:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8110.
00000100:00000200:0.0:1788971660.565397:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189552512, offset 0
00000400:00000200:0.0:1788971660.565401:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971660.565405:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971660.565409:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971660.565411:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971660.565414:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971660.565416:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971660.565418:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971660.565421:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [307:307:255:4294967295] 192.168.203.150@tcp LPNI seq info [307:307:7:4294967295]
00000400:00000200:0.0:1788971660.565425:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971660.565430:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971660.565435:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971660.565438:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f900.
00000800:00000200:0.0:1788971660.565442:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971660.565446:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971660.565449:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971660.565454:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971660.565472:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4472800.
00000400:00000200:0.0:1788971660.565475:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971660.565477:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971660.565480:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000.
00000100:00000200:0.0:1788971660.565483:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552384, portal 10
00000400:00000010:0.0:1788971660.565487:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8cc0.
00000100:00000200:0.0:1788971660.565489:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189552384, offset 0
00000400:00000200:0.0:1788971660.565492:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971660.565502:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971660.565505:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971660.565507:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971660.565509:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971660.565511:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971660.565513:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971660.565516:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [308:308:254:4294967295] 192.168.203.150@tcp LPNI seq info [308:308:6:4294967295]
00000400:00000200:0.0:1788971660.565520:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971660.565526:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971660.565530:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971660.565533:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000.
00000800:00000200:0.0:1788971660.565537:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971660.565541:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971660.565543:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971660.565548:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971660.565986:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971660.565997:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f500.
00000400:00000200:2.0:1788971660.566006:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971660.566015:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971660.566024:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088
00000400:00000010:2.0:1788971660.566027:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088.
00000100:00000200:2.0:1788971660.566040:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a89500 x1875872189552256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971660.566080:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971660.566083:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f900.
00000400:00000200:2.0:1788971660.566086:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971660.566099:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971660.566103:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8110
00000400:00000010:2.0:1788971660.566105:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8110.
00000100:00000200:2.0:1788971660.566111:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88e00 x1875872189552512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971660.566152:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971660.566154:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000.
00000400:00000200:2.0:1788971660.566157:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971660.566160:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971660.566163:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8cc0
00000400:00000010:2.0:1788971660.566165:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8cc0.
00000100:00000200:2.0:1788971660.566170:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88a80 x1875872189552384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971660.575103:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.575113:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971660.575122:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.575126:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.575131:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.575135:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809680
00000400:00000200:2.0:1788971660.575140:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.575145:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xfb5 [1] + 224
00000400:00000200:2.0:1788971660.575158:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971660.575163:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971660.575177:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a89500 x1875872189552256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971660.578055:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:0.0:1788971660.578079:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:0.0:1788971660.578082:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971660.578095:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a89500 x1875872189552256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971660.578105:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a89500 x1875872189552256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971660.578137:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4e00.
02000000:00000010:0.0:1788971660.578145:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060ff00.
00000100:00000010:0.0:1788971660.578149:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500.
00000400:00000200:2.0:1788971660.582447:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.582457:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971660.583079:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583085:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583092:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583109:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809780
00000400:00000200:2.0:1788971660.583115:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583120:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xfbd [1] + 224
00000400:00000200:2.0:1788971660.583134:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971660.583138:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971660.583151:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88e00 x1875872189552512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971660.583284:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583288:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971660.583293:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583296:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583299:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583301:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809700
00000400:00000200:2.0:1788971660.583316:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971660.583320:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xfc5 [1] + 224
00000400:00000200:2.0:1788971660.583333:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971660.583337:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000010:0.0:1788971660.583341:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:0.0:1788971660.583348:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8bb0
00000400:00000010:0.0:1788971660.583350:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8bb0.
00000100:00000200:0.0:1788971660.583359:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88e00 x1875872189552512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971660.583361:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88a80 x1875872189552384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971660.583394:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88e00 x1875872189552512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971660.583423:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc5e00.
02000000:00000010:0.0:1788971660.583430:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100.
00000100:00000010:0.0:1788971660.583433:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88e00.
00000400:00000010:0.0:1788971660.583439:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971660.583441:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000
00000400:00000010:0.0:1788971660.583442:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000.
00000100:00000200:0.0:1788971660.583447:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88a80 x1875872189552384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971660.583454:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88a80 x1875872189552384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971676 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971660.583465:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4472800.
02000000:00000010:0.0:1788971660.583468:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f400.
00000100:00000010:0.0:1788971660.583470:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80.
00000100:00000010:3.0:1788971661.585937:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d80.
00000100:00000010:2.0:1788971661.585937:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5cc0.
00000100:00000010:3.0:1788971665.683675:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791f80.
02000000:00000010:3.0:1788971665.683699:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600.
00000100:00000010:3.0:1788971665.683742:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80.
02000000:00000010:3.0:1788971665.683748:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00.
00000100:00000010:3.0:1788971665.683755:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000.
02000000:00000010:3.0:1788971665.683758:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200.
02000000:00000010:3.0:1788971665.683790:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800.
00000400:00000200:3.0:1788971665.683799:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971665.683803:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160.
00000400:00000010:3.0:1788971665.683810:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8.
02000000:00000010:2.0:1788971665.684084:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1285400.
00000400:00000200:2.0:1788971665.684090:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971665.684094:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160.
00000100:00000200:3.0:1788971665.684128:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552768, portal 10
00000400:00000010:3.0:1788971665.684137:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0.
00000400:00000010:2.0:1788971665.687130:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788971665.687147:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552640, portal 25
00000400:00000010:2.0:1788971665.687154:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971665.687158:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189552640, offset 0
00000400:00000200:2.0:1788971665.687170:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971665.687182:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.687189:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971665.687209:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971665.687214:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971665.687217:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971665.687221:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971665.687225:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [309:309:256:4294967295] 192.168.203.150@tcp LPNI seq info [309:309:8:4294967295]
00000400:00000200:2.0:1788971665.687230:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.687345:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971665.687352:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971665.687360:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c100.
00000800:00000200:2.0:1788971665.687368:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971665.687377:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971665.687382:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971665.687519:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971665.687525:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e8a0.
00000100:00000010:2.0:1788971665.687537:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed20.
02000000:00000010:2.0:1788971665.687558:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343e00.
00000400:00000200:2.0:1788971665.687565:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971665.687570:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126688.
00000400:00000010:2.0:1788971665.687574:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971665.687577:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189552896, portal 10
00000400:00000010:2.0:1788971665.687582:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971665.687583:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189552896, offset 0
00000400:00000200:2.0:1788971665.687588:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971665.687591:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.687595:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971665.687597:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971665.687599:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971665.687601:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971665.687604:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971665.687606:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [310:310:255:4294967295] 192.168.203.150@tcp LPNI seq info [310:310:7:4294967295]
00000400:00000200:2.0:1788971665.687611:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.687623:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971665.687627:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971665.687630:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c800.
00000800:00000200:2.0:1788971665.687634:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971665.687638:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971665.687641:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971665.687646:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971665.687708:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971665.687712:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c100.
00000400:00000200:2.0:1788971665.687720:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971665.687724:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971665.687729:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0
00000400:00000010:2.0:1788971665.687731:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788971665.687740:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8791f80 x1875872189552640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971665.687811:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971665.687831:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c800.
00000400:00000200:2.0:1788971665.687835:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971665.687838:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971665.687841:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18
00000400:00000010:2.0:1788971665.687843:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971665.687848:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8790000 x1875872189552896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971665.687963:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189552768, offset 0
00000400:00000200:3.0:1788971665.687971:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971665.687975:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971665.687979:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971665.687981:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971665.687984:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971665.687986:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971665.687995:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971665.687998:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [311:311:256:4294967295] 192.168.203.150@tcp LPNI seq info [311:311:8:4294967295]
00000400:00000200:3.0:1788971665.688002:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971665.688009:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971665.688013:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788971665.688017:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300.
00000800:00000200:3.0:1788971665.688021:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788971665.688025:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971665.688028:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971665.688052:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971665.689568:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971665.689575:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300.
00000400:00000200:2.0:1788971665.689582:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971665.689589:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971665.689596:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0
00000400:00000010:2.0:1788971665.689598:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0.
00000100:00000200:2.0:1788971665.689608:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8792d80 x1875872189552768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971665.703018:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.703028:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971665.703049:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.703054:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.703059:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.703062:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809800
00000400:00000200:2.0:1788971665.703068:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.703075:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0xfd5 [1] + 224
00000400:00000200:2.0:1788971665.703093:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971665.703098:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971665.703112:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8791f80 x1875872189552640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971665.703615:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160.
00000400:00000200:3.0:1788971665.703622:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198
00000400:00000010:3.0:1788971665.703624:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198.
00000100:00000200:3.0:1788971665.703631:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8791f80 x1875872189552640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971665.703642:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8791f80 x1875872189552640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971665.703670:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1285400.
02000000:00000010:3.0:1788971665.703677:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600.
00000100:00000010:3.0:1788971665.703682:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791f80.
00000400:00000200:2.0:1788971665.704973:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.704981:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971665.704987:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.704990:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.704995:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.704998:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809880
00000400:00000200:2.0:1788971665.705003:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.705009:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xfcd [1] + 224
00000400:00000200:2.0:1788971665.705018:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971665.705022:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971665.705045:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8792d80 x1875872189552768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971665.705118:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.705121:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971665.705126:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.705128:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.705151:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.705154:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809900
00000400:00000200:2.0:1788971665.705157:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971665.705161:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xfe5 [1] + 224
00000400:00000200:2.0:1788971665.705170:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971665.705173:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971665.705180:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8790000 x1875872189552896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971665.705210:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126688.
00000400:00000200:2.0:1788971665.705215:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:2.0:1788971665.705217:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971665.705222:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8790000 x1875872189552896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971665.705229:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8790000 x1875872189552896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971665.705384:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343e00.
02000000:00000010:2.0:1788971665.705390:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200.
00000100:00000010:2.0:1788971665.705518:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000.
00000400:00000010:3.0:1788971665.705570:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160.
00000400:00000200:3.0:1788971665.705575:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8
00000400:00000010:3.0:1788971665.705577:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8.
00000100:00000200:3.0:1788971665.705584:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8792d80 x1875872189552768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971665.705592:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8792d80 x1875872189552768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971681 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971665.705609:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800.
02000000:00000010:3.0:1788971665.705614:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00.
00000100:00000010:3.0:1788971665.705624:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80.
00000100:00000010:2.0:1788971666.706319:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e8a0.
00000100:00000010:3.0:1788971666.706412:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed20.
00000100:00000010:0.0:1788971670.802081:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80.
02000000:00000010:0.0:1788971670.802104:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f400.
00000100:00000010:0.0:1788971670.802143:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88e00.
02000000:00000010:0.0:1788971670.802149:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100.
00000100:00000010:0.0:1788971670.802155:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500.
02000000:00000010:0.0:1788971670.802158:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060ff00.
02000000:00000010:0.0:1788971670.802217:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4472800.
00000400:00000200:0.0:1788971670.802237:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971670.802241:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971670.802248:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000.
00000100:00000200:0.0:1788971670.802255:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553152, portal 10
00000400:00000010:0.0:1788971670.802260:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8bb0.
00000100:00000200:0.0:1788971670.802263:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189553152, offset 0
00000400:00000200:0.0:1788971670.802272:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971670.802294:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971670.802301:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971670.802306:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971670.802309:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971670.802312:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971670.802315:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971670.802319:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [312:312:256:4294967295] 192.168.203.150@tcp LPNI seq info [312:312:8:4294967295]
00000400:00000200:0.0:1788971670.802323:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971670.802332:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971670.802338:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971670.802344:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000.
00000800:00000200:0.0:1788971670.802350:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971670.802357:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971670.802527:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1
02000000:00000010:1.0:1788971670.802617:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46cb800.
00000400:00000200:1.0:1788971670.802623:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971670.802626:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000010:1.0:1788971670.802630:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525b28.
00000100:00000200:1.0:1788971670.802633:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553024, portal 25
00000400:00000010:1.0:1788971670.802637:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220.
00000100:00000200:1.0:1788971670.802639:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189553024, offset 0
00000400:00000200:1.0:1788971670.802644:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971670.802648:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971670.802652:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971670.802654:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971670.802657:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971670.802659:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971670.802661:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971670.802664:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [313:313:255:4294967295] 192.168.203.150@tcp LPNI seq info [313:313:7:4294967295]
00000400:00000200:1.0:1788971670.802668:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971670.802674:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971670.802678:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971670.802682:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f000.
00000800:00000200:1.0:1788971670.802686:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971670.802690:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971670.802693:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971670.802713:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971670.802718:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c58a0.
00000100:00000010:0.0:1788971670.802728:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5f60.
00000400:00000200:1.0:1788971670.804922:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:1.0:1788971670.804961:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46ca800.
00000400:00000200:1.0:1788971670.804968:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971670.804989:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000010:1.0:1788971670.805004:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525088.
00000100:00000200:1.0:1788971670.805011:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553280, portal 10
00000400:00000010:1.0:1788971670.805019:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18.
00000100:00000200:1.0:1788971670.805022:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189553280, offset 0
00000400:00000200:1.0:1788971670.805031:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971670.805039:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971670.805045:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971670.805050:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971670.805054:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971670.805057:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971670.805060:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971670.805064:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [314:314:254:4294967295] 192.168.203.150@tcp LPNI seq info [314:314:6:4294967295]
00000400:00000200:1.0:1788971670.805068:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971670.805077:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971670.805083:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971670.805088:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f900.
00000800:00000200:1.0:1788971670.805094:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971670.805101:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971670.805104:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971670.805110:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971670.807039:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971670.807046:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000.
00000400:00000200:2.0:1788971670.807054:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971670.807061:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971670.807068:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8bb0
00000400:00000010:2.0:1788971670.807071:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8bb0.
00000100:00000200:2.0:1788971670.807081:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88e00 x1875872189553152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971670.807106:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971670.807108:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f000.
00000400:00000200:2.0:1788971670.807111:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971670.807126:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971670.807129:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220
00000400:00000010:2.0:1788971670.807131:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220.
00000100:00000200:2.0:1788971670.807137:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88a80 x1875872189553024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971670.811065:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971670.811074:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f900.
00000400:00000200:2.0:1788971670.811082:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971670.811090:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971670.811096:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18
00000400:00000010:2.0:1788971670.811099:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18.
00000100:00000200:2.0:1788971670.811111:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a89500 x1875872189553280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971670.817441:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.817452:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971670.817463:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.817807:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.817831:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.817836:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809980
00000400:00000200:2.0:1788971670.817842:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.817848:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1005 [1] + 224
00000400:00000200:2.0:1788971670.817863:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971670.817867:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971670.817881:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88a80 x1875872189553024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971670.818009:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818013:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971670.818018:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818021:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818024:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818040:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809a00
00000400:00000200:2.0:1788971670.818044:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818048:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0xffd [1] + 224
00000400:00000200:2.0:1788971670.818056:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971670.818059:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971670.818066:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88e00 x1875872189553152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971670.818095:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818099:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971670.818102:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818104:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818107:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818109:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809a80
00000400:00000200:2.0:1788971670.818113:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971670.818116:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x100d [1] + 224
00000400:00000200:2.0:1788971670.818137:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971670.818141:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971670.818147:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a89500 x1875872189553280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:0.0:1788971670.818334:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:0.0:1788971670.818339:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000
00000400:00000010:0.0:1788971670.818341:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000.
00000100:00000200:0.0:1788971670.818348:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88e00 x1875872189553152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:0.0:1788971670.818356:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88e00 x1875872189553152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:0.0:1788971670.818388:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4472800.
02000000:00000010:0.0:1788971670.818395:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100.
00000100:00000010:0.0:1788971670.818399:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88e00.
00000400:00000010:1.0:1788971670.821869:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000200:1.0:1788971670.821882:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525b28
00000400:00000010:1.0:1788971670.821885:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525b28.
00000100:00000200:1.0:1788971670.821897:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88a80 x1875872189553024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971670.821908:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88a80 x1875872189553024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971670.821934:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000200:1.0:1788971670.821936:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525088
00000400:00000010:1.0:1788971670.821938:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525088.
00000100:00000200:1.0:1788971670.821957:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a89500 x1875872189553280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971670.821965:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a89500 x1875872189553280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971686 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971670.821983:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46cb800.
02000000:00000010:1.0:1788971670.821989:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f400.
00000100:00000010:1.0:1788971670.822003:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80.
02000000:00000010:1.0:1788971670.822007:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46ca800.
02000000:00000010:1.0:1788971670.822010:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060ff00.
00000100:00000010:1.0:1788971670.822012:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500.
00000100:00000010:3.0:1788971671.825954:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5f60.
00000100:00000010:2.0:1788971671.826408:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c58a0.
00000100:00000010:2.0:1788971675.922200:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f800.
02000000:00000010:2.0:1788971675.922221:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c800.
00000100:00000010:2.0:1788971675.922257:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000.
02000000:00000010:2.0:1788971675.922263:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c100.
00000100:00000010:2.0:1788971675.922270:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80.
02000000:00000010:2.0:1788971675.922275:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c600.
02000000:00000010:2.0:1788971675.922301:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c1343e00.
02000000:00000010:3.0:1788971675.922302:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800.
00000400:00000200:2.0:1788971675.922312:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971675.922313:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971675.922319:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126688.
00000400:00000010:3.0:1788971675.922320:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160.
00000400:00000010:3.0:1788971675.922326:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8.
00000100:00000200:3.0:1788971675.922333:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553408, portal 25
00000400:00000010:2.0:1788971675.922338:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000400:00000010:3.0:1788971675.922341:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873000.
00000100:00000200:2.0:1788971675.922344:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553536, portal 10
00000100:00000200:3.0:1788971675.922346:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189553408, offset 0
00000400:00000010:2.0:1788971675.922352:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18.
00000400:00000200:3.0:1788971675.922358:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000100:00000200:2.0:1788971675.922364:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189553536, offset 0
00000400:00000200:3.0:1788971675.922370:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.922376:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788971675.922381:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971675.922386:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971675.922390:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971675.922394:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971675.922397:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971675.922400:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [315:315:256:4294967295] 192.168.203.150@tcp LPNI seq info [315:315:8:4294967295]
00000400:00000200:3.0:1788971675.922405:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971675.922414:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000400:00000200:2.0:1788971675.922423:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788971675.922424:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971675.922430:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000800:00000010:3.0:1788971675.922432:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00.
00000400:00000200:2.0:1788971675.922435:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971675.922439:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000800:00000200:3.0:1788971675.922443:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000400:00000200:2.0:1788971675.922445:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971675.922449:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000800:00000200:3.0:1788971675.922455:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:2.0:1788971675.922455:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [316:316:255:4294967295] 192.168.203.150@tcp LPNI seq info [316:316:7:4294967295]
00000800:00000200:3.0:1788971675.922462:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971675.922463:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.922473:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971675.922478:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971675.922482:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c200.
00000400:00000200:3.0:1788971675.922483:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971675.922489:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000100:00000010:3.0:1788971675.922491:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c6534180.
00000800:00000200:2.0:1788971675.922496:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971675.922499:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c200 type 1, nob 320 niov 1 nkiov 1
00000100:00000010:3.0:1788971675.922505:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c6534b40.
00000400:00000200:2.0:1788971675.922509:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788971675.922520:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b200.
00000400:00000200:3.0:1788971675.922526:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788971675.922528:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18.
00000400:00000010:3.0:1788971675.922539:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0.
00000100:00000200:3.0:1788971675.922556:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553664, portal 10
00000800:00000200:2.0:1788971675.922562:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000400:00000010:3.0:1788971675.922563:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440.
00000100:00000200:3.0:1788971675.922569:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189553664, offset 0
00000800:00000010:2.0:1788971675.922569:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00.
00000400:00000200:2.0:1788971675.922578:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971675.922579:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971675.922587:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788971675.922588:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971675.922595:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788971675.922598:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788971675.922600:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788971675.922603:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788971675.922605:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788971675.922608:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [317:317:254:4294967295] 192.168.203.150@tcp LPNI seq info [317:317:6:4294967295]
00000400:00000200:3.0:1788971675.922611:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971675.922617:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788971675.922622:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971675.922623:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873000
00000800:00000010:3.0:1788971675.922626:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600.
00000400:00000010:2.0:1788971675.922628:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873000.
00000800:00000200:3.0:1788971675.922634:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000100:00000200:2.0:1788971675.922640:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f800 x1875872189553408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788971675.922642:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788971675.922648:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788971675.922658:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971675.922672:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971675.922676:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c200.
00000400:00000200:2.0:1788971675.922681:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971675.922692:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971675.922695:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18
00000400:00000010:2.0:1788971675.922697:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788971675.922701:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840c000 x1875872189553536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971675.922772:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971675.922775:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600.
00000400:00000200:2.0:1788971675.922778:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971675.922781:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971675.922783:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440
00000400:00000010:2.0:1788971675.922785:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440.
00000100:00000200:2.0:1788971675.922789:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840fb80 x1875872189553664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971675.930512:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930522:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971675.930532:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930537:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930556:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930560:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809b00
00000400:00000200:2.0:1788971675.930566:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930572:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1015 [1] + 224
00000400:00000200:2.0:1788971675.930585:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971675.930590:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971675.930604:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f800 x1875872189553408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971675.930695:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930700:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971675.930705:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930708:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930710:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930739:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809b80
00000400:00000200:2.0:1788971675.930744:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930748:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x101d [1] + 224
00000400:00000200:2.0:1788971675.930756:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971675.930760:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971675.930767:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840c000 x1875872189553536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971675.930802:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930806:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971675.930810:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930812:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930862:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930865:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809c00
00000400:00000200:2.0:1788971675.930868:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971675.930872:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1035 [1] + 224
00000400:00000200:2.0:1788971675.930881:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971675.930885:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971675.930891:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840fb80 x1875872189553664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971675.930931:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126688.
00000400:00000200:2.0:1788971675.930936:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:2.0:1788971675.930938:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788971675.930944:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840c000 x1875872189553536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971675.930952:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840c000 x1875872189553536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971675.930994:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c1343e00.
02000000:00000010:2.0:1788971675.931000:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c100.
00000100:00000010:2.0:1788971675.931006:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000.
00000400:00000010:3.0:1788971675.936180:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160.
00000400:00000200:3.0:1788971675.936193:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8
00000400:00000010:3.0:1788971675.936196:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8.
00000100:00000200:3.0:1788971675.936210:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f800 x1875872189553408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971675.936221:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f800 x1875872189553408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788971675.936256:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18.
00000400:00000200:3.0:1788971675.936259:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0
00000400:00000010:3.0:1788971675.936260:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0.
00000100:00000200:3.0:1788971675.936265:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840fb80 x1875872189553664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788971675.936290:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840fb80 x1875872189553664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971691 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788971675.936312:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800.
02000000:00000010:3.0:1788971675.936321:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c800.
00000100:00000010:3.0:1788971675.936327:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f800.
02000000:00000010:3.0:1788971675.936330:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b200.
02000000:00000010:3.0:1788971675.936334:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c600.
00000100:00000010:3.0:1788971675.936336:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80.
00000100:00000010:3.0:1788971676.947932:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c6534b40.
00000100:00000010:2.0:1788971676.950106:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c6534180.
00000001:02000400:0.0:1788971679.741178:0:110243:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_280: @@@@@@ FAIL: Failed to unload modules before test execution
00000100:00000010:0.0:1788971681.042391:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88e00.
02000000:00000010:0.0:1788971681.042425:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100.
00000100:00000010:0.0:1788971681.042467:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500.
02000000:00000010:0.0:1788971681.042482:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f900.
00000100:00000010:0.0:1788971681.042489:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80.
02000000:00000010:0.0:1788971681.042493:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f500.
02000000:00000010:0.0:1788971681.042520:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46ca400.
00000400:00000200:0.0:1788971681.042530:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971681.042533:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f20.
00000400:00000010:0.0:1788971681.042539:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000.
00000100:00000200:0.0:1788971681.042545:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553920, portal 10
00000400:00000010:0.0:1788971681.042551:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880.
00000100:00000200:0.0:1788971681.042554:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189553920, offset 0
00000400:00000200:0.0:1788971681.042563:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971681.042571:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971681.042578:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971681.042583:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971681.042587:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971681.042591:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971681.042594:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971681.042598:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [318:318:256:4294967295] 192.168.203.150@tcp LPNI seq info [318:318:8:4294967295]
00000400:00000200:0.0:1788971681.042602:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971681.042613:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971681.042619:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971681.042624:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00.
00000800:00000200:0.0:1788971681.042630:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971681.042638:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971681.042642:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971681.042661:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:0.0:1788971681.042666:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c55a0.
00000100:00000010:0.0:1788971681.042677:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c57e0.
02000000:00000010:0.0:1788971681.042700:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46cb600.
00000400:00000200:0.0:1788971681.042711:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971681.042713:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4318.
00000400:00000010:0.0:1788971681.042717:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8bb0.
00000100:00000200:0.0:1788971681.042721:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189553792, portal 25
00000400:00000010:0.0:1788971681.042724:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8cc0.
00000100:00000200:0.0:1788971681.042726:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189553792, offset 0
00000400:00000200:0.0:1788971681.042731:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971681.042734:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971681.042737:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971681.042740:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971681.042742:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971681.042745:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971681.042747:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971681.042750:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [319:319:255:4294967295] 192.168.203.150@tcp LPNI seq info [319:319:7:4294967295]
00000400:00000200:0.0:1788971681.042753:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971681.042760:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971681.042764:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971681.042767:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f400.
00000800:00000200:0.0:1788971681.042771:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971681.042775:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971681.042778:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971681.042782:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:0.0:1788971681.042794:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46cb800.
00000400:00000200:0.0:1788971681.042797:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:0.0:1788971681.042799:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4b00.
00000400:00000010:0.0:1788971681.042802:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8110.
00000100:00000200:0.0:1788971681.042805:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189554048, portal 10
00000400:00000010:0.0:1788971681.042809:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088.
00000100:00000200:0.0:1788971681.042811:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189554048, offset 0
00000400:00000200:0.0:1788971681.042841:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:0.0:1788971681.042844:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971681.042866:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:0.0:1788971681.042869:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:0.0:1788971681.042872:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:0.0:1788971681.042874:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:0.0:1788971681.042877:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:0.0:1788971681.042880:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [320:320:254:4294967295] 192.168.203.150@tcp LPNI seq info [320:320:6:4294967295]
00000400:00000200:0.0:1788971681.042884:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:0.0:1788971681.042890:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:0.0:1788971681.042895:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:0.0:1788971681.042898:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000.
00000800:00000200:0.0:1788971681.042902:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:0.0:1788971681.042906:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:0.0:1788971681.042909:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:0.0:1788971681.042914:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971681.043148:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971681.043153:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00.
00000400:00000200:2.0:1788971681.043159:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971681.043164:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971681.043170:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880
00000400:00000010:2.0:1788971681.043172:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880.
00000100:00000200:2.0:1788971681.043182:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a89500 x1875872189553920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971681.043219:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971681.043222:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f400.
00000400:00000200:2.0:1788971681.043225:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971681.043228:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971681.043231:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8cc0
00000400:00000010:2.0:1788971681.043233:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8cc0.
00000100:00000200:2.0:1788971681.043238:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88e00 x1875872189553792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971681.043303:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971681.043306:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000.
00000400:00000200:2.0:1788971681.043309:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971681.043313:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971681.043316:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088
00000400:00000010:2.0:1788971681.043318:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088.
00000100:00000200:2.0:1788971681.043322:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73d1a88a80 x1875872189554048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971681.053387:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053397:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971681.053405:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053410:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053415:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053419:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887809c80
00000400:00000200:2.0:1788971681.053424:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053443:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x104d [1] + 224
00000400:00000200:2.0:1788971681.053456:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971681.053461:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971681.053475:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88e00 x1875872189553792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971681.053513:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053518:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971681.053522:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053524:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053527:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053530:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809d00
00000400:00000200:2.0:1788971681.053533:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053536:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1045 [1] + 224
00000400:00000200:2.0:1788971681.053542:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971681.053545:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971681.053550:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a89500 x1875872189553920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971681.053563:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053566:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971681.053569:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053572:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053574:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053576:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887809d80
00000400:00000200:2.0:1788971681.053579:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971681.053583:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1055 [1] + 224
00000400:00000200:2.0:1788971681.053590:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971681.053594:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971681.053599:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73d1a88a80 x1875872189554048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971681.053798:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4318.
00000400:00000200:1.0:1788971681.053804:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8bb0
00000400:00000010:1.0:1788971681.053807:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8bb0.
00000100:00000200:1.0:1788971681.053842:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88e00 x1875872189553792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971681.053852:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88e00 x1875872189553792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971681.053874:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4b00.
00000400:00000200:1.0:1788971681.053878:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8110
00000400:00000010:1.0:1788971681.053879:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8110.
00000100:00000200:1.0:1788971681.053883:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a88a80 x1875872189554048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971681.053890:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a88a80 x1875872189554048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971681.053904:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f20.
00000400:00000200:1.0:1788971681.053907:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000
00000400:00000010:1.0:1788971681.053908:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000.
00000100:00000200:1.0:1788971681.053912:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73d1a89500 x1875872189553920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971681.053942:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73d1a89500 x1875872189553920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971697 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971681.053960:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46cb600.
02000000:00000010:1.0:1788971681.053966:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100.
00000100:00000010:1.0:1788971681.053971:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88e00.
02000000:00000010:1.0:1788971681.053975:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46cb800.
02000000:00000010:1.0:1788971681.053977:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f500.
00000100:00000010:1.0:1788971681.053979:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80.
02000000:00000010:1.0:1788971681.053981:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46ca400.
02000000:00000010:1.0:1788971681.053984:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f900.
00000100:00000010:1.0:1788971681.053985:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500.
Debug log: 4365 lines, 4365 kept, 0 dropped, 0 bad.