00000100:00000010:1.0F:1788971829.522139:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788971829.522286:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f800. 00000100:00000010:1.0:1788971829.522323:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788971829.522328:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00. 00000100:00000010:1.0:1788971829.522334:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700. 02000000:00000010:1.0:1788971829.522338:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fb00. 02000000:00000010:1.0:1788971829.522381:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc5e00. 00000400:00000200:1.0:1788971829.522393:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971829.522399:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788971829.522407:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788971829.522416:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564928, portal 25 02000000:00000010:0.0F:1788971829.522571:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003400. 00000400:00000200:0.0:1788971829.522583:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971829.522587:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4dc0. 00000400:00000010:0.0:1788971829.522591:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a86e8. 00000100:00000200:0.0:1788971829.522594:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565056, portal 10 00000400:00000010:0.0:1788971829.522601:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8440. 00000100:00000200:0.0:1788971829.522605:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189565056, offset 0 00000400:00000200:0.0:1788971829.522617:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971829.522628:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971829.522640:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971829.522646:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971829.522651:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971829.522655:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971829.522658:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971829.522662:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [405:405:256:4294967295] 192.168.203.150@tcp LPNI seq info [405:405:8:4294967295] 00000400:00000200:0.0:1788971829.522668:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971829.522679:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971829.522687:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971829.522695:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be400. 00000800:00000200:0.0:1788971829.522702:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971829.522712:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971829.522718:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971829.522734:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788971829.522740:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d20. 00000100:00000010:0.0:1788971829.522753:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d80. 00000400:00000010:1.0:1788971829.523593:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788971829.523605:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189564928, offset 0 00000400:00000200:1.0:1788971829.523614:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971829.523621:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971829.523626:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971829.523630:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971829.523634:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971829.523636:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971829.523639:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971829.523642:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [406:406:255:4294967295] 192.168.203.150@tcp LPNI seq info [406:406:7:4294967295] 00000400:00000200:1.0:1788971829.523646:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971829.523655:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971829.523660:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971829.523664:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f000. 00000800:00000200:1.0:1788971829.523670:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971829.523676:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971829.523680:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971829.523686:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788971829.523709:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4e00. 00000400:00000200:1.0:1788971829.523713:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971829.523716:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971829.523728:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788971829.523732:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565184, portal 10 00000400:00000010:1.0:1788971829.523738:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000. 00000100:00000200:1.0:1788971829.523740:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189565184, offset 0 00000400:00000200:1.0:1788971829.523745:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971829.523748:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971829.523752:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971829.523755:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971829.523757:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971829.523759:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971829.523761:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971829.523764:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [407:407:254:4294967295] 192.168.203.150@tcp LPNI seq info [407:407:6:4294967295] 00000400:00000200:1.0:1788971829.523767:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971829.523773:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971829.523777:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971829.523780:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f900. 00000800:00000200:1.0:1788971829.523784:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971829.523787:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971829.523790:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971829.523795:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0F:1788971829.524123:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971829.524135:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be400. 00000400:00000200:2.0:1788971829.524140:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971829.524146:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971829.524164:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8440 00000400:00000010:2.0:1788971829.524167:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8440. 00000100:00000200:2.0:1788971829.524180:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189565056/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971829.524209:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971829.524211:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f000. 00000400:00000200:2.0:1788971829.524215:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971829.524218:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971829.524220:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:2.0:1788971829.524222:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:2.0:1788971829.524227:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189564928/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971829.524307:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971829.524310:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f900. 00000400:00000200:2.0:1788971829.524314:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971829.524317:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971829.524320:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000 00000400:00000010:2.0:1788971829.524322:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000. 00000100:00000200:2.0:1788971829.524327:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f0700 x1875872189565184/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971829.528900:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.528914:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971829.528924:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.528929:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.528936:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.528940:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780c880 00000400:00000200:2.0:1788971829.528946:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.528953:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x13f5 [1] + 224 00000400:00000200:2.0:1788971829.528986:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971829.528992:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971829.529841:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189565056/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971829.529895:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529899:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971829.529904:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529907:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529912:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529915:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780c900 00000400:00000200:2.0:1788971829.529920:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529925:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x13fd [1] + 224 00000400:00000200:2.0:1788971829.529934:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971829.529938:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971829.529947:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f0700 x1875872189565184/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971829.529966:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529970:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971829.529974:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529976:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529979:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529982:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780c800 00000400:00000200:2.0:1788971829.529986:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971829.529990:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x13ed [1] + 224 00000400:00000010:1.0:1788971829.530712:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4dc0. 00000400:00000200:1.0:1788971829.530720:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a86e8 00000400:00000010:1.0:1788971829.530723:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a86e8. 00000100:00000200:1.0:1788971829.530733:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189565056/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971829.530743:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189565056/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971829.530787:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003400. 02000000:00000010:1.0:1788971829.530794:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00. 00000100:00000010:1.0:1788971829.530801:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000400:00000010:1.0:1788971829.530858:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971829.530862:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:1.0:1788971829.530863:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788971829.530880:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f0700 x1875872189565184/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971829.530889:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f0700 x1875872189565184/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971829.530906:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4e00. 02000000:00000010:1.0:1788971829.530910:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fb00. 00000100:00000010:1.0:1788971829.530913:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700. 00000400:00000200:2.0:1788971829.530965:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971829.530971:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971829.530981:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189564928/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971829.531325:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971829.531329:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:1.0:1788971829.531331:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788971829.531336:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189564928/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971829.531344:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189564928/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971845 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971829.531359:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc5e00. 02000000:00000010:1.0:1788971829.531364:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f800. 00000100:00000010:1.0:1788971829.531367:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 00000001:02000400:2.0:1788971829.826514:0:112281:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 300: packaged LNet UAPI headers can be compiled ========================================================== 12:37:08 (1788971828) 00000020:00000010:0.0:1788971830.182414:0:112307:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780fc0. 00000020:00000010:0.0:1788971830.182509:0:112307:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780fc0. 00000020:00000010:0.0:1788971830.253448:0:112312:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971830.253596:0:112312:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000100:00000010:2.0:1788971830.546034:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d20. 00000100:00000010:3.0F:1788971830.546301:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d80. 00000020:00000010:0.0:1788971831.305549:0:112316:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971831.305640:0:112316:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971832.345407:0:112320:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971832.345497:0:112320:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000020:00000010:3.0:1788971833.383948:0:112324:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5360. 00000020:00000010:3.0:1788971833.384038:0:112324:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5360. 00000020:00000010:2.0:1788971834.439600:0:112329:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4eac0. 00000020:00000010:2.0:1788971834.439691:0:112329:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4eac0. 00000100:00000010:0.0:1788971834.642023:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788971834.642044:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09beb00. 00000100:00000010:0.0:1788971834.642081:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788971834.642089:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be100. 00000100:00000010:0.0:1788971834.642096:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b100. 02000000:00000010:0.0:1788971834.642099:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be000. 02000000:00000010:1.0:1788971834.642285:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc5600. 00000400:00000200:1.0:1788971834.642296:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971834.642301:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788971834.642307:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788971834.642314:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565312, portal 25 00000400:00000010:1.0:1788971834.642319:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788971834.642323:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189565312, offset 0 00000400:00000200:1.0:1788971834.642331:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971834.642340:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971834.642348:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971834.642354:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971834.642371:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971834.642375:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971834.642377:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971834.642381:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [408:408:256:4294967295] 192.168.203.150@tcp LPNI seq info [408:408:8:4294967295] 00000400:00000200:1.0:1788971834.642386:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971834.642396:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971834.642402:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971834.642407:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f800. 00000800:00000200:1.0:1788971834.642413:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971834.642420:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971834.642424:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971834.642489:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971834.642495:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467c00. 00000100:00000010:1.0:1788971834.642504:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467ba0. 02000000:00000010:1.0:1788971834.642515:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4e00. 00000400:00000200:1.0:1788971834.642519:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971834.642521:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971834.642525:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971834.642527:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565568, portal 10 00000400:00000010:1.0:1788971834.642532:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525440. 00000100:00000200:1.0:1788971834.642535:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189565568, offset 0 00000400:00000200:1.0:1788971834.642539:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971834.642543:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971834.642546:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971834.642549:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971834.642552:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971834.642554:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971834.642556:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971834.642559:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [409:409:255:4294967295] 192.168.203.150@tcp LPNI seq info [409:409:7:4294967295] 00000400:00000200:1.0:1788971834.642562:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971834.642568:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971834.642579:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971834.642582:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0fb00. 00000800:00000200:1.0:1788971834.642585:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971834.642590:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971834.642592:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971834.642597:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788971834.642615:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc5e00. 00000400:00000200:1.0:1788971834.642618:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971834.642621:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000010:1.0:1788971834.642625:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788971834.642628:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565440, portal 10 00000400:00000010:1.0:1788971834.642631:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788971834.642633:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189565440, offset 0 00000400:00000200:1.0:1788971834.642638:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971834.642641:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971834.642645:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971834.642647:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971834.642650:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971834.642652:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971834.642654:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971834.642656:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [410:410:254:4294967295] 192.168.203.150@tcp LPNI seq info [410:410:6:4294967295] 00000400:00000200:1.0:1788971834.642660:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971834.642665:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971834.642668:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971834.642672:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0fa00. 00000800:00000200:1.0:1788971834.642674:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971834.642678:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971834.642680:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0fa00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971834.642685:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971834.648025:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971834.648035:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f800. 00000400:00000200:2.0:1788971834.648043:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971834.648052:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971834.648059:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:2.0:1788971834.648061:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:2.0:1788971834.648073:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a300 x1875872189565312/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971834.648100:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971834.648102:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0fb00. 00000400:00000200:2.0:1788971834.648105:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971834.648109:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971834.648111:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525440 00000400:00000010:2.0:1788971834.648113:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525440. 00000100:00000200:2.0:1788971834.648119:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b100 x1875872189565568/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971834.653491:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971834.653504:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0fa00. 00000400:00000200:2.0:1788971834.653513:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971834.653522:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971834.653529:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:2.0:1788971834.653532:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000100:00000200:2.0:1788971834.653544:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189565440/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971834.660930:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.660940:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971834.660948:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.660966:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.660972:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.660976:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780c980 00000400:00000200:2.0:1788971834.660981:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.660987:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1405 [1] + 224 00000400:00000200:2.0:1788971834.661000:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971834.661005:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971834.661019:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a300 x1875872189565312/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971834.661176:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971834.661196:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:1.0:1788971834.661198:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788971834.661205:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a300 x1875872189565312/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971834.661214:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a300 x1875872189565312/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971834.661240:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc5600. 02000000:00000010:1.0:1788971834.661246:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09beb00. 00000100:00000010:1.0:1788971834.661250:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a300. 00000400:00000200:2.0:1788971834.666707:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666717:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971834.666726:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666730:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666735:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666740:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ca80 00000400:00000200:2.0:1788971834.666746:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666752:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x140d [1] + 224 00000400:00000200:2.0:1788971834.666764:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971834.666769:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971834.666784:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b100 x1875872189565568/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971834.666858:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666862:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971834.666867:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666869:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666872:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666875:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ca00 00000400:00000200:2.0:1788971834.666878:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971834.666882:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1415 [1] + 224 00000400:00000200:2.0:1788971834.666891:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971834.666907:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971834.666914:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189565440/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971834.667524:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000200:1.0:1788971834.667531:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525088 00000400:00000010:1.0:1788971834.667534:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788971834.667541:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189565440/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971834.667551:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189565440/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971834.667579:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971834.667582:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:1.0:1788971834.667584:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971834.667588:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b100 x1875872189565568/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971834.667595:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b100 x1875872189565568/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971850 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971834.667609:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc5e00. 02000000:00000010:1.0:1788971834.667617:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be100. 00000100:00000010:1.0:1788971834.667622:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 02000000:00000010:1.0:1788971834.667625:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4e00. 02000000:00000010:1.0:1788971834.667627:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be000. 00000100:00000010:1.0:1788971834.667629:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b100. 00000020:00000010:3.0:1788971835.476641:0:112333:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5360. 00000020:00000010:3.0:1788971835.476739:0:112333:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5360. 00000100:00000010:3.0:1788971835.666076:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467ba0. 00000100:00000010:2.0:1788971835.670344:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467c00. 00000020:00000010:3.0:1788971836.526775:0:112337:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5360. 00000020:00000010:3.0:1788971836.527291:0:112337:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5360. 00000020:00000010:1.0:1788971837.578713:0:112341:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971837.579072:0:112341:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:2.0:1788971838.645520:0:112345:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4eac0. 00000020:00000010:2.0:1788971838.645634:0:112345:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4eac0. 00000020:00000010:3.0:1788971839.756209:0:112349:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5360. 00000020:00000010:3.0:1788971839.756302:0:112349:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5360. 00000100:00000010:3.0:1788971839.764455:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788971839.764475:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518db00. 00000100:00000010:3.0:1788971839.764503:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791180. 02000000:00000010:3.0:1788971839.764508:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d200. 00000100:00000010:3.0:1788971839.764514:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791880. 02000000:00000010:3.0:1788971839.764518:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d500. 02000000:00000010:3.0:1788971839.764544:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f200. 00000400:00000200:3.0:1788971839.764554:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971839.764559:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436cb8. 00000400:00000010:3.0:1788971839.764565:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873088. 00000100:00000200:3.0:1788971839.764573:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565696, portal 25 00000400:00000010:3.0:1788971839.764578:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788971839.764581:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189565696, offset 0 00000400:00000200:3.0:1788971839.764590:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971839.764598:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.764605:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971839.764610:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971839.764614:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971839.764617:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971839.764620:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971839.764624:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [411:411:256:4294967295] 192.168.203.150@tcp LPNI seq info [411:411:8:4294967295] 00000400:00000200:3.0:1788971839.764629:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.764638:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971839.764653:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971839.764658:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d43ac300. 00000800:00000200:3.0:1788971839.764664:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971839.764671:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971839.764675:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d43ac300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971839.764689:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971839.764694:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09960. 00000100:00000010:3.0:1788971839.764703:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09060. 02000000:00000010:3.0:1788971839.764713:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000. 00000400:00000200:3.0:1788971839.764717:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971839.764720:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436840. 00000400:00000010:3.0:1788971839.764723:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48732a8. 00000100:00000200:3.0:1788971839.764726:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565952, portal 10 00000400:00000010:3.0:1788971839.764730:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788971839.764732:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189565952, offset 0 00000400:00000200:3.0:1788971839.764736:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971839.764740:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.764744:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971839.764746:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971839.764769:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971839.764772:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971839.764774:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971839.764778:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [412:412:255:4294967295] 192.168.203.150@tcp LPNI seq info [412:412:7:4294967295] 00000400:00000200:3.0:1788971839.764782:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.764787:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971839.764791:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971839.764805:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788971839.764809:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971839.764834:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971839.764837:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971839.764850:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788971839.764869:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600. 00000400:00000200:3.0:1788971839.764873:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971839.764876:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788971839.764879:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788971839.764882:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189565824, portal 10 00000400:00000010:3.0:1788971839.764886:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788971839.764889:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189565824, offset 0 00000400:00000200:3.0:1788971839.764893:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971839.764897:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.764900:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971839.764903:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971839.764905:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971839.764907:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971839.764909:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971839.764912:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [413:413:254:4294967295] 192.168.203.150@tcp LPNI seq info [413:413:6:4294967295] 00000400:00000200:3.0:1788971839.764916:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.764920:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971839.764924:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971839.764927:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788971839.764931:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971839.764935:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971839.764937:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971839.764942:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971839.764988:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971839.764991:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d43ac300. 00000400:00000200:3.0:1788971839.764996:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971839.765001:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971839.765006:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000010:3.0:1788971839.765009:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788971839.765016:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189565696/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971839.765045:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971839.765047:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:3.0:1788971839.765050:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971839.765054:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971839.765056:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:3.0:1788971839.765058:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788971839.765062:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791880 x1875872189565952/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971839.765123:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971839.765126:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:3.0:1788971839.765129:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971839.765133:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971839.765135:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0 00000400:00000010:3.0:1788971839.765136:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788971839.765141:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791180 x1875872189565824/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971839.768699:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.768709:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971839.768716:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.768721:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.768726:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.768730:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780cb00 00000400:00000200:2.0:1788971839.768734:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.768740:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x141d [1] + 224 00000400:00000200:2.0:1788971839.768754:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971839.768759:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971839.768774:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189565696/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971839.768851:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436cb8. 00000400:00000200:2.0:1788971839.768856:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873088 00000400:00000010:2.0:1788971839.768859:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873088. 00000100:00000200:2.0:1788971839.768865:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189565696/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971839.768873:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189565696/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971839.768907:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f200. 02000000:00000010:2.0:1788971839.768913:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518db00. 00000100:00000010:2.0:1788971839.768918:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 00000400:00000200:3.0:1788971839.782199:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.782211:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971839.782221:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.782226:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.782232:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.782236:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780cc00 00000400:00000200:3.0:1788971839.782241:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971839.782247:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x142d [1] + 224 00000400:00000200:2.0:1788971839.788016:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971839.788048:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971839.788067:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791880 x1875872189565952/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971839.788108:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.788114:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971839.788121:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.788125:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.788129:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.788133:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780cb80 00000400:00000200:2.0:1788971839.788138:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971839.788144:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x143d [1] + 224 00000400:00000200:2.0:1788971839.788156:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971839.788160:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971839.788166:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791180 x1875872189565824/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971839.788548:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788971839.788554:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:3.0:1788971839.788556:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788971839.788563:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791180 x1875872189565824/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971839.788572:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791180 x1875872189565824/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971839.789566:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436840. 00000400:00000200:2.0:1788971839.789573:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48732a8 00000400:00000010:2.0:1788971839.789574:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48732a8. 00000100:00000200:2.0:1788971839.789583:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791880 x1875872189565952/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971839.790052:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791880 x1875872189565952/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971855 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971839.790076:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600. 02000000:00000010:2.0:1788971839.790084:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d200. 00000100:00000010:2.0:1788971839.790089:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791180. 02000000:00000010:2.0:1788971839.790094:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000. 02000000:00000010:2.0:1788971839.790096:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d500. 00000100:00000010:2.0:1788971839.790098:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791880. 00000100:00000010:3.0:1788971840.792412:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09060. 00000100:00000010:2.0:1788971840.802975:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09960. 00000020:00000010:1.0:1788971840.829730:0:112353:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971840.830114:0:112353:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:0.0:1788971841.916093:0:112358:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971841.916174:0:112358:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000020:00000010:2.0:1788971842.981265:0:112362:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4eac0. 00000020:00000010:2.0:1788971842.981341:0:112362:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4eac0. 00000020:00000010:1.0:1788971844.043516:0:112366:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971844.043599:0:112366:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:3.0:1788971844.884228:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792a00. 02000000:00000010:3.0:1788971844.884251:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788971844.884286:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788971844.884291:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788971844.884298:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791500. 02000000:00000010:3.0:1788971844.884302:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 02000000:00000010:2.0:1788971844.887210:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094ae00. 00000400:00000200:2.0:1788971844.887228:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971844.887232:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126840. 00000400:00000010:2.0:1788971844.887239:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971844.887246:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566080, portal 25 00000400:00000010:2.0:1788971844.887252:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550. 00000100:00000200:2.0:1788971844.887255:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189566080, offset 0 00000400:00000200:2.0:1788971844.887264:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971844.887275:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971844.887294:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971844.887299:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971844.887303:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971844.887307:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971844.887310:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971844.887313:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [414:414:256:4294967295] 192.168.203.150@tcp LPNI seq info [414:414:8:4294967295] 00000400:00000200:2.0:1788971844.887317:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971844.887326:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971844.887332:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971844.887337:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383a00. 00000800:00000200:2.0:1788971844.887343:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971844.887350:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971844.887354:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971844.887388:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971844.887394:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ef00. 00000100:00000010:2.0:1788971844.887404:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ec00. 02000000:00000010:2.0:1788971844.887415:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000400:00000200:2.0:1788971844.887419:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971844.887421:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126e18. 00000400:00000010:2.0:1788971844.887424:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788971844.887441:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566336, portal 10 00000400:00000010:2.0:1788971844.887447:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8. 00000100:00000200:2.0:1788971844.887449:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189566336, offset 0 00000400:00000200:2.0:1788971844.887454:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971844.887458:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971844.887461:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971844.887464:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971844.887466:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971844.887469:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971844.887471:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971844.887474:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [415:415:255:4294967295] 192.168.203.150@tcp LPNI seq info [415:415:7:4294967295] 00000400:00000200:2.0:1788971844.887477:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971844.887484:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971844.887488:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971844.887492:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383300. 00000800:00000200:2.0:1788971844.887497:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971844.887502:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971844.887505:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971844.887510:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971844.887530:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000010:2.0:1788971844.887534:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b000. 00000800:00000010:3.0:1788971844.887536:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383a00. 00000400:00000200:2.0:1788971844.887540:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971844.887544:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10. 00000400:00000200:3.0:1788971844.887545:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1788971844.887548:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000400:00000200:3.0:1788971844.887553:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971844.887556:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566208, portal 10 00000400:00000200:3.0:1788971844.887559:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550 00000400:00000010:2.0:1788971844.887561:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000400:00000010:3.0:1788971844.887563:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550. 00000100:00000200:3.0:1788971844.887574:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792a00 x1875872189566080/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971860 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971844.887651:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971844.887654:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383300. 00000400:00000200:3.0:1788971844.887657:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971844.887660:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971844.887769:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189566208, offset 0 00000400:00000200:2.0:1788971844.887776:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971844.887780:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971844.887784:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971844.887786:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971844.887788:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971844.887795:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971844.887797:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971844.887799:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [416:416:255:4294967295] 192.168.203.150@tcp LPNI seq info [416:416:7:4294967295] 00000400:00000200:2.0:1788971844.887803:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971844.887808:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971844.887812:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971844.887841:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383500. 00000800:00000200:2.0:1788971844.887845:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971844.887849:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971844.887852:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971844.887858:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903252:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8 00000400:00000010:3.0:1788971844.903262:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8. 00000100:00000200:3.0:1788971844.903279:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791500 x1875872189566336/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971844.903317:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903322:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971844.903328:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903332:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903336:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903340:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780cc80 00000400:00000200:3.0:1788971844.903344:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903662:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x144d [1] + 224 00000400:00000200:3.0:1788971844.903675:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971844.903680:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971844.903690:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792a00 x1875872189566080/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971844.903770:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971844.903774:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383500. 00000400:00000200:3.0:1788971844.903778:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971844.903781:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971844.903784:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:3.0:1788971844.903786:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:3.0:1788971844.903792:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790e00 x1875872189566208/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971844.903809:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903850:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971844.903856:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903859:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903862:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903865:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780cd80 00000400:00000200:3.0:1788971844.903869:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.903873:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x145d [1] + 224 00000400:00000200:3.0:1788971844.903882:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971844.903886:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971844.903891:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791500 x1875872189566336/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971844.903924:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126840. 00000400:00000200:3.0:1788971844.903929:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:3.0:1788971844.903931:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:3.0:1788971844.903937:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792a00 x1875872189566080/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971844.903958:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792a00 x1875872189566080/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971844.903980:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126e18. 00000400:00000200:3.0:1788971844.903983:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:3.0:1788971844.903985:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:3.0:1788971844.903989:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791500 x1875872189566336/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971844.903997:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791500 x1875872189566336/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971844.904014:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094ae00. 02000000:00000010:3.0:1788971844.904022:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788971844.904026:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792a00. 02000000:00000010:3.0:1788971844.904029:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:3.0:1788971844.904033:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788971844.904035:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791500. 00000400:00000200:3.0:1788971844.912773:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.912783:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971844.912793:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.912797:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.912802:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.912806:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780cd00 00000400:00000200:3.0:1788971844.912812:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971844.912853:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x146d [1] + 224 00000400:00000200:3.0:1788971844.912867:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971844.912871:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971844.912884:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790e00 x1875872189566208/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971844.915885:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10. 00000400:00000200:2.0:1788971844.915899:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788971844.915902:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971844.915915:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790e00 x1875872189566208/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971844.915927:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790e00 x1875872189566208/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971860 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971844.915959:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b000. 02000000:00000010:2.0:1788971844.915968:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788971844.915973:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790e00. 00000020:00000010:2.0:1788971845.135512:0:112370:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4eac0. 00000020:00000010:2.0:1788971845.135604:0:112370:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4eac0. 00000100:00000010:2.0:1788971845.905999:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ef00. 00000100:00000010:3.0:1788971845.906141:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ec00. 00000020:00000010:2.0:1788971846.217261:0:112374:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4eac0. 00000020:00000010:2.0:1788971846.217433:0:112374:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4eac0. 00000020:00000010:3.0:1788971847.284635:0:112378:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971847.284722:0:112378:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000020:00000010:1.0:1788971848.358322:0:112382:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971848.358418:0:112382:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971849.398126:0:112387:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971849.398222:0:112387:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:3.0:1788971850.002224:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791500. 02000000:00000010:3.0:1788971850.002244:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788971850.002268:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792a00. 02000000:00000010:3.0:1788971850.002272:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788971850.002277:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788971850.002281:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200. 02000000:00000010:2.0:1788971850.002792:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800. 00000400:00000200:2.0:1788971850.002803:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971850.002807:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10. 00000400:00000010:2.0:1788971850.002837:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971850.002858:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566464, portal 25 00000400:00000010:2.0:1788971850.002864:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971850.002867:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189566464, offset 0 00000400:00000200:2.0:1788971850.002876:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971850.002884:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971850.002892:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971850.002897:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971850.002901:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971850.002904:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971850.002907:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971850.002915:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [417:417:256:4294967295] 192.168.203.150@tcp LPNI seq info [417:417:8:4294967295] 00000400:00000200:2.0:1788971850.002920:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971850.002928:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971850.002935:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971850.002941:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383d00. 00000800:00000200:2.0:1788971850.002947:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971850.002965:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971850.002969:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971850.002988:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971850.002994:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ef00. 00000100:00000010:2.0:1788971850.003004:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e9c0. 02000000:00000010:2.0:1788971850.003014:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094aa00. 00000400:00000200:2.0:1788971850.003018:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971850.003021:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971850.003025:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788971850.003028:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566720, portal 10 00000400:00000010:2.0:1788971850.003032:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788971850.003034:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189566720, offset 0 00000400:00000200:2.0:1788971850.003039:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971850.003042:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971850.003045:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971850.003055:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971850.003057:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971850.003059:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971850.003061:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971850.003075:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [418:418:255:4294967295] 192.168.203.150@tcp LPNI seq info [418:418:7:4294967295] 00000400:00000200:2.0:1788971850.003080:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971850.003086:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971850.003091:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971850.003095:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383500. 00000800:00000200:2.0:1788971850.003100:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971850.003104:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971850.003106:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971850.003116:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971850.003135:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788971850.003138:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971850.003141:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971850.003144:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971850.003146:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566592, portal 10 00000400:00000010:2.0:1788971850.003150:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b330. 00000100:00000200:2.0:1788971850.003152:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189566592, offset 0 00000400:00000200:2.0:1788971850.003156:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971850.003159:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971850.003162:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971850.003164:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971850.003166:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971850.003168:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971850.003169:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971850.003172:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [419:419:254:4294967295] 192.168.203.150@tcp LPNI seq info [419:419:6:4294967295] 00000400:00000200:2.0:1788971850.003176:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971850.003181:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971850.003193:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971850.003197:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383300. 00000800:00000200:2.0:1788971850.003201:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971850.003205:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971850.003207:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971850.003212:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971850.003416:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971850.003420:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383d00. 00000400:00000200:3.0:1788971850.003427:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971850.003432:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971850.003446:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:3.0:1788971850.003449:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:3.0:1788971850.003459:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791500 x1875872189566464/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971850.003482:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971850.003484:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383500. 00000400:00000200:3.0:1788971850.003487:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971850.003490:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971850.003492:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:3.0:1788971850.003493:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:3.0:1788971850.003497:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790a80 x1875872189566720/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971850.003566:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971850.003569:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383300. 00000400:00000200:3.0:1788971850.003572:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971850.003575:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971850.003578:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b330 00000400:00000010:3.0:1788971850.003579:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b330. 00000100:00000200:3.0:1788971850.003584:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792a00 x1875872189566592/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971850.007083:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007092:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971850.007101:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007106:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007111:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007115:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780cf00 00000400:00000200:3.0:1788971850.007120:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007126:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x148d [1] + 224 00000400:00000200:3.0:1788971850.007138:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971850.007142:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971850.007156:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790a80 x1875872189566720/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971850.007198:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007202:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971850.007206:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007208:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007211:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007214:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ce80 00000400:00000200:3.0:1788971850.007217:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.007220:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x149d [1] + 224 00000400:00000200:3.0:1788971850.007228:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971850.007231:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971850.007236:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792a00 x1875872189566592/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971850.009587:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:2.0:1788971850.009599:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971850.009602:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971850.009614:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792a00 x1875872189566592/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971850.009627:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792a00 x1875872189566592/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971850.009658:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971850.009661:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788971850.009663:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788971850.009668:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790a80 x1875872189566720/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971850.009676:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790a80 x1875872189566720/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971850.009693:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:2.0:1788971850.009701:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:2.0:1788971850.009706:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792a00. 02000000:00000010:2.0:1788971850.009710:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094aa00. 02000000:00000010:2.0:1788971850.009712:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200. 00000100:00000010:2.0:1788971850.009714:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790a80. 00000400:00000200:3.0:1788971850.011147:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.011154:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971850.011162:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.011166:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.011185:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.011190:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780ce00 00000400:00000200:3.0:1788971850.011195:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971850.011201:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x147d [1] + 224 00000400:00000200:3.0:1788971850.011215:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971850.011219:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971850.011230:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791500 x1875872189566464/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971850.011667:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10. 00000400:00000200:2.0:1788971850.011673:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788971850.011675:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971850.011689:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791500 x1875872189566464/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971850.011698:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791500 x1875872189566464/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971866 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971850.011737:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800. 02000000:00000010:2.0:1788971850.011744:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788971850.011749:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791500. 00000020:00000010:1.0:1788971850.468807:0:112391:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971850.468922:0:112391:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:2.0:1788971851.026284:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ef00. 00000100:00000010:3.0:1788971851.026410:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e9c0. 00000020:00000010:1.0:1788971851.522130:0:112395:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971851.522218:0:112395:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971852.598975:0:112399:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971852.599066:0:112399:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971853.692401:0:112404:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971853.692500:0:112404:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971854.755534:0:112408:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971854.755623:0:112408:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:2.0:1788971855.124373:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971855.124396:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c5383a00. 00000100:00000010:2.0:1788971855.124437:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788971855.124442:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c5383e00. 00000100:00000010:2.0:1788971855.124448:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80. 02000000:00000010:2.0:1788971855.124453:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c5383300. 02000000:00000010:2.0:1788971855.124483:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788971855.124490:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971855.124494:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10. 00000400:00000010:2.0:1788971855.124500:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971855.124507:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566976, portal 10 00000400:00000010:2.0:1788971855.124512:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788971855.124515:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189566976, offset 0 00000400:00000200:2.0:1788971855.124523:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971855.124531:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971855.124538:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971855.124542:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971855.124547:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971855.124550:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971855.124553:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971855.124556:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [420:420:256:4294967295] 192.168.203.150@tcp LPNI seq info [420:420:8:4294967295] 00000400:00000200:2.0:1788971855.124560:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971855.124568:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971855.124586:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971855.124600:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383500. 00000800:00000200:2.0:1788971855.124607:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971855.124613:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971855.124618:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971855.124630:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971855.124635:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ef00. 00000100:00000010:2.0:1788971855.124645:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e240. 02000000:00000010:2.0:1788971855.124666:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b600. 00000400:00000200:2.0:1788971855.124669:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971855.124671:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971855.124674:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971855.124677:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189566848, portal 25 00000400:00000010:2.0:1788971855.124680:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bdd0. 00000100:00000200:2.0:1788971855.124682:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189566848, offset 0 00000400:00000200:2.0:1788971855.124696:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971855.124700:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971855.124703:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971855.124705:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971855.124707:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971855.124709:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971855.124711:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971855.124714:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [421:421:255:4294967295] 192.168.203.150@tcp LPNI seq info [421:421:7:4294967295] 00000400:00000200:2.0:1788971855.124718:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971855.124723:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971855.124726:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971855.124729:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383d00. 00000800:00000200:2.0:1788971855.124732:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971855.124735:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971855.124737:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971855.124746:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971855.124756:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094ba00. 00000400:00000200:2.0:1788971855.124759:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971855.124761:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971855.124764:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971855.124766:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567104, portal 10 00000400:00000010:2.0:1788971855.124770:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788971855.124772:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189567104, offset 0 00000400:00000200:2.0:1788971855.124775:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971855.124778:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971855.124781:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971855.124783:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971855.124785:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971855.124787:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971855.124789:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971855.124792:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [422:422:254:4294967295] 192.168.203.150@tcp LPNI seq info [422:422:6:4294967295] 00000400:00000200:2.0:1788971855.124795:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971855.124800:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971855.124803:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971855.124806:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d500. 00000800:00000200:2.0:1788971855.124809:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971855.124812:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971855.124829:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971855.124833:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971855.125007:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971855.125011:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383500. 00000400:00000200:3.0:1788971855.125019:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971855.125025:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971855.125030:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:3.0:1788971855.125032:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:3.0:1788971855.125041:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f480 x1875872189566976/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971855.125063:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971855.125065:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383d00. 00000400:00000200:3.0:1788971855.125067:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971855.125071:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971855.125073:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bdd0 00000400:00000010:3.0:1788971855.125075:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bdd0. 00000100:00000200:3.0:1788971855.125079:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189566848/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971855.125154:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971855.125157:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d500. 00000400:00000200:3.0:1788971855.125160:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971855.125163:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971855.125166:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:3.0:1788971855.125167:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:3.0:1788971855.125179:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ca80 x1875872189567104/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971855.132220:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.132230:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971855.132238:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.132243:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.132247:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.132251:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780cf80 00000400:00000200:3.0:1788971855.132257:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.132263:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x14bd [1] + 224 00000400:00000200:3.0:1788971855.132276:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971855.132280:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971855.132306:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189566848/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971855.132529:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971855.132535:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971855.132538:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971855.132545:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189566848/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971855.132554:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189566848/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971855.132598:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b600. 02000000:00000010:2.0:1788971855.132606:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c5383a00. 00000100:00000010:2.0:1788971855.132610:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000400:00000200:3.0:1788971855.139271:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.139344:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971855.139356:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.139360:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.139366:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.139370:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d000 00000400:00000200:3.0:1788971855.139375:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.139381:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x14ad [1] + 224 00000400:00000200:3.0:1788971855.139396:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971855.139400:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971855.139415:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f480 x1875872189566976/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971855.139501:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10. 00000400:00000200:2.0:1788971855.139509:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788971855.139512:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971855.139521:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f480 x1875872189566976/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971855.139532:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f480 x1875872189566976/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971855.139564:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:2.0:1788971855.139572:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c5383e00. 00000100:00000010:2.0:1788971855.139577:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480. 00000400:00000200:3.0:1788971855.147613:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.147621:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971855.147629:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.147645:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.147652:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.147655:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d080 00000400:00000200:3.0:1788971855.147660:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971855.147666:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x14cd [1] + 224 00000400:00000200:3.0:1788971855.147678:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971855.147683:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971855.147709:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ca80 x1875872189567104/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971855.150105:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:2.0:1788971855.150120:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788971855.150123:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971855.150134:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ca80 x1875872189567104/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971855.150146:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ca80 x1875872189567104/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971871 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971855.150178:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094ba00. 02000000:00000010:2.0:1788971855.150185:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c5383300. 00000100:00000010:2.0:1788971855.150190:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80. 00000020:00000010:1.0:1788971855.796299:0:112412:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971855.796398:0:112412:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:3.0:1788971856.145978:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e240. 00000100:00000010:2.0:1788971856.145979:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ef00. 00000020:00000010:0.0:1788971856.850325:0:112416:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971856.850436:0:112416:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971857.916371:0:112420:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971857.916943:0:112420:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000020:00000010:1.0:1788971858.988339:0:112424:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971858.988433:0:112424:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:0.0:1788971860.024522:0:112428:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971860.024612:0:112428:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000100:00000010:2.0:1788971860.242133:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80. 02000000:00000010:2.0:1788971860.242156:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d200. 00000100:00000010:2.0:1788971860.242204:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788971860.242209:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518db00. 00000100:00000010:2.0:1788971860.242215:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971860.242233:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d500. 02000000:00000010:3.0:1788971860.242306:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9fa00. 00000400:00000200:3.0:1788971860.242316:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971860.242320:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788971860.242326:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788971860.242331:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567232, portal 25 00000400:00000010:3.0:1788971860.242336:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788971860.242338:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189567232, offset 0 00000400:00000200:3.0:1788971860.242346:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971860.242353:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.242361:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971860.242365:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971860.242369:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971860.242373:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971860.242375:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971860.242379:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [423:423:256:4294967295] 192.168.203.150@tcp LPNI seq info [423:423:8:4294967295] 00000400:00000200:3.0:1788971860.242382:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.242391:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971860.242396:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971860.242401:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788971860.242407:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971860.242421:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971860.242426:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971860.242448:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971860.242453:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09ae0. 00000100:00000010:3.0:1788971860.242463:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09d20. 02000000:00000010:3.0:1788971860.242474:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600. 00000400:00000200:3.0:1788971860.242479:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971860.242482:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688. 00000400:00000010:3.0:1788971860.242486:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788971860.242489:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567488, portal 10 00000400:00000010:3.0:1788971860.242493:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788971860.242495:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189567488, offset 0 00000400:00000200:3.0:1788971860.242499:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971860.242503:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.242507:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971860.242509:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971860.242512:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971860.242514:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971860.242516:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971860.242519:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [424:424:255:4294967295] 192.168.203.150@tcp LPNI seq info [424:424:7:4294967295] 00000400:00000200:3.0:1788971860.242523:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.242529:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971860.242533:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971860.242536:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788971860.242540:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971860.242544:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971860.242547:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971860.242552:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788971860.242568:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000. 00000400:00000200:3.0:1788971860.242571:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971860.242579:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788971860.242583:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788971860.242586:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567360, portal 10 00000400:00000010:3.0:1788971860.242591:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788971860.242592:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189567360, offset 0 00000400:00000200:3.0:1788971860.242597:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971860.242601:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.242605:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971860.242607:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971860.242610:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971860.242612:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971860.242614:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971860.242617:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [425:425:254:4294967295] 192.168.203.150@tcp LPNI seq info [425:425:6:4294967295] 00000400:00000200:3.0:1788971860.242620:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.242626:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971860.242630:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971860.242633:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300. 00000800:00000200:3.0:1788971860.242637:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971860.242641:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971860.242644:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971860.242648:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971860.242856:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971860.242861:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788971860.242868:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971860.242873:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971860.242879:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0 00000400:00000010:2.0:1788971860.242881:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0. 00000100:00000200:2.0:1788971860.242890:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ca80 x1875872189567232/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971860.242914:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971860.242921:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788971860.242925:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971860.242928:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971860.242931:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000010:2.0:1788971860.242933:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000100:00000200:2.0:1788971860.242938:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189567488/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971860.243341:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971860.243344:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300. 00000400:00000200:2.0:1788971860.243349:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971860.243353:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971860.243357:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873198 00000400:00000010:2.0:1788971860.243359:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873198. 00000100:00000200:2.0:1788971860.243365:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f480 x1875872189567360/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971860.245995:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246004:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971860.246014:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246019:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246025:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246029:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d200 00000400:00000200:3.0:1788971860.246034:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246039:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x14ed [1] + 224 00000400:00000200:3.0:1788971860.246051:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971860.246061:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971860.246088:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189567488/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971860.246200:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246204:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971860.246526:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246531:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246534:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246537:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d180 00000400:00000200:3.0:1788971860.246540:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246544:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x14fd [1] + 224 00000400:00000200:3.0:1788971860.246553:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971860.246556:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971860.246563:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f480 x1875872189567360/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971860.246580:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246583:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971860.246588:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246591:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246594:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246596:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780d100 00000400:00000200:3.0:1788971860.246600:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971860.246604:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x14dd [1] + 224 00000400:00000200:3.0:1788971860.246612:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971860.246616:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971860.246622:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ca80 x1875872189567232/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971860.246699:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688. 00000400:00000200:2.0:1788971860.246706:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:2.0:1788971860.246708:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000100:00000200:2.0:1788971860.246715:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189567488/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971860.246724:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189567488/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971860.246764:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600. 02000000:00000010:2.0:1788971860.246770:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d500. 00000100:00000010:2.0:1788971860.246774:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000400:00000010:2.0:1788971860.246782:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:2.0:1788971860.246785:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8 00000400:00000010:2.0:1788971860.246786:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8. 00000100:00000200:2.0:1788971860.246792:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f480 x1875872189567360/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971860.246801:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f480 x1875872189567360/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971860.246839:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:2.0:1788971860.246842:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:2.0:1788971860.246844:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:2.0:1788971860.246848:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ca80 x1875872189567232/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971860.246856:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ca80 x1875872189567232/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971876 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971860.246867:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000. 02000000:00000010:2.0:1788971860.246870:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518db00. 00000100:00000010:2.0:1788971860.246873:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788971860.246876:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9fa00. 02000000:00000010:2.0:1788971860.246878:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d200. 00000100:00000010:2.0:1788971860.246880:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80. 00000020:00000010:1.0:1788971861.056422:0:112433:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971861.056514:0:112433:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:3.0:1788971861.266041:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09d20. 00000100:00000010:2.0:1788971861.266435:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09ae0. 00000020:00000010:1.0:1788971862.100210:0:112437:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971862.100297:0:112437:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:0.0:1788971863.171938:0:112441:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971863.172037:0:112441:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000020:00000010:1.0:1788971864.236641:0:112445:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971864.246026:0:112445:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:0.0:1788971865.292690:0:112449:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971865.292917:0:112449:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000100:00000010:3.0:1788971865.362335:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791500. 02000000:00000010:3.0:1788971865.362357:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788971865.362406:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788971865.362411:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300. 00000100:00000010:3.0:1788971865.362416:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792a00. 02000000:00000010:3.0:1788971865.362419:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 02000000:00000010:2.0:1788971865.362632:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788971865.362642:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971865.362646:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971865.362652:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971865.362658:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567616, portal 25 00000400:00000010:2.0:1788971865.362668:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971865.362672:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189567616, offset 0 00000400:00000200:2.0:1788971865.362686:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971865.362694:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971865.362703:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971865.362708:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971865.362713:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971865.362717:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971865.362720:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971865.362723:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [426:426:256:4294967295] 192.168.203.150@tcp LPNI seq info [426:426:8:4294967295] 00000400:00000200:2.0:1788971865.362727:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971865.362735:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971865.362740:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971865.362745:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d200. 00000800:00000200:2.0:1788971865.362752:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971865.362761:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971865.362766:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971865.362782:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971865.362787:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ef00. 00000100:00000010:2.0:1788971865.362797:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e2a0. 02000000:00000010:2.0:1788971865.362806:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a600. 00000400:00000200:2.0:1788971865.362809:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971865.362811:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10. 00000400:00000010:2.0:1788971865.362847:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971865.362851:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567872, portal 10 00000400:00000010:2.0:1788971865.362855:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788971865.362858:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189567872, offset 0 00000400:00000200:2.0:1788971865.362863:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971865.362866:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971865.362870:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971865.362873:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971865.362882:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971865.362884:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971865.362886:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971865.362889:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [427:427:255:4294967295] 192.168.203.150@tcp LPNI seq info [427:427:7:4294967295] 00000400:00000200:2.0:1788971865.362894:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971865.362903:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971865.362908:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971865.362912:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518db00. 00000800:00000200:2.0:1788971865.362917:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971865.362921:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971865.362924:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971865.362929:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971865.362950:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094ba00. 00000400:00000200:2.0:1788971865.362954:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971865.362957:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971865.362961:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788971865.362964:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189567744, portal 10 00000400:00000010:2.0:1788971865.362968:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788971865.362970:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189567744, offset 0 00000800:00000200:3.0:1788971865.362980:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:2.0:1788971865.362980:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971865.362990:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:3.0:1788971865.362993:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d200. 00000400:00000200:2.0:1788971865.362997:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971865.363003:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971865.363004:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971865.363009:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971865.363016:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971865.363023:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971865.363024:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971865.363033:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [428:428:254:4294967295] 192.168.203.150@tcp LPNI seq info [428:428:6:4294967295] 00000400:00000200:2.0:1788971865.363037:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971865.363085:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971865.363091:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971865.363100:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d500. 00000800:00000200:2.0:1788971865.363104:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971865.363109:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971865.363112:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971865.363117:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.366368:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:3.0:1788971865.366381:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:3.0:1788971865.366402:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791500 x1875872189567616/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971865.366476:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971865.366481:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518db00. 00000400:00000200:3.0:1788971865.366486:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971865.366491:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971865.366495:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b5d8 00000400:00000010:3.0:1788971865.366497:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b5d8. 00000100:00000200:3.0:1788971865.366503:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792a00 x1875872189567872/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971865.368058:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971865.368070:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d500. 00000400:00000200:3.0:1788971865.368077:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971865.368086:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971865.368096:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:3.0:1788971865.368099:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:3.0:1788971865.368111:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790a80 x1875872189567744/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971865.373020:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373034:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971865.373045:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373069:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373076:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373081:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780d280 00000400:00000200:3.0:1788971865.373088:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373095:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x150d [1] + 224 00000400:00000200:3.0:1788971865.373109:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971865.373115:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971865.373145:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791500 x1875872189567616/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971865.373188:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373192:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971865.373196:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373198:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373201:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373204:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d380 00000400:00000200:3.0:1788971865.373207:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.373212:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x151d [1] + 224 00000400:00000200:3.0:1788971865.373221:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971865.373224:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971865.373232:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792a00 x1875872189567872/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971865.373515:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:2.0:1788971865.373528:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788971865.373530:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971865.373544:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791500 x1875872189567616/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971865.373555:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791500 x1875872189567616/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971865.373585:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10. 00000400:00000200:2.0:1788971865.373588:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971865.373589:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971865.373594:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792a00 x1875872189567872/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971865.373603:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792a00 x1875872189567872/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971865.373632:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:2.0:1788971865.373640:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00. 00000100:00000010:2.0:1788971865.373646:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791500. 02000000:00000010:2.0:1788971865.373650:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a600. 02000000:00000010:2.0:1788971865.373652:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788971865.373654:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792a00. 00000400:00000200:3.0:1788971865.376267:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.376281:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971865.376291:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.376297:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.376304:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.376308:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d300 00000400:00000200:3.0:1788971865.376314:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971865.376320:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x152d [1] + 224 00000400:00000200:3.0:1788971865.377776:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971865.377806:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971865.377844:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790a80 x1875872189567744/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971865.377928:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971865.377940:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:2.0:1788971865.377943:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788971865.377955:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790a80 x1875872189567744/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971865.377987:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790a80 x1875872189567744/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971881 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971865.378019:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094ba00. 02000000:00000010:2.0:1788971865.378026:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300. 00000100:00000010:2.0:1788971865.378030:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790a80. 00000020:00000010:0.0:1788971866.331085:0:112453:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971866.331171:0:112453:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000100:00000010:3.0:1788971866.385910:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e2a0. 00000100:00000010:2.0:1788971866.386066:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ef00. 00000020:00000010:2.0:1788971867.357710:0:112457:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e820. 00000020:00000010:2.0:1788971867.361580:0:112457:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e820. 00000020:00000010:3.0:1788971868.402567:0:112462:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971868.402671:0:112462:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000020:00000010:2.0:1788971869.446361:0:112466:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e820. 00000020:00000010:2.0:1788971869.446442:0:112466:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e820. 00000100:00000010:2.0:1788971870.483043:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80. 02000000:00000010:2.0:1788971870.483069:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d500. 00000100:00000010:2.0:1788971870.483105:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788971870.483110:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518db00. 00000100:00000010:2.0:1788971870.483116:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971870.483120:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d200. 02000000:00000010:2.0:1788971870.483147:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788971870.483156:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971870.483160:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971870.483167:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788971870.483173:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568000, portal 25 00000400:00000010:2.0:1788971870.483178:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971870.483181:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189568000, offset 0 00000400:00000200:2.0:1788971870.483190:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971870.483197:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971870.483204:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971870.483209:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971870.483213:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971870.483216:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971870.483219:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971870.483230:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [429:429:256:4294967295] 192.168.203.150@tcp LPNI seq info [429:429:8:4294967295] 02000000:00000010:3.0:1788971870.483233:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000. 00000400:00000200:2.0:1788971870.483235:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.483238:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971870.483241:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68. 00000400:00000200:2.0:1788971870.483243:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000400:00000010:3.0:1788971870.483244:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873550. 00000100:00000200:3.0:1788971870.483247:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568128, portal 10 00000800:00000200:2.0:1788971870.483250:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000010:3.0:1788971870.483251:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788971870.483253:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189568128, offset 0 00000800:00000010:2.0:1788971870.483255:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f900. 00000400:00000200:3.0:1788971870.483258:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000800:00000200:2.0:1788971870.483261:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:3.0:1788971870.483262:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.483265:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971870.483268:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:2.0:1788971870.483268:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:3.0:1788971870.483270:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971870.483272:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000800:00000200:2.0:1788971870.483272:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971870.483274:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971870.483277:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [430:430:255:4294967295] 192.168.203.150@tcp LPNI seq info [430:430:7:4294967295] 00000400:00000200:3.0:1788971870.483280:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971870.483285:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.483286:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000100:00000010:2.0:1788971870.483289:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ef00. 00000800:00000200:3.0:1788971870.483290:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971870.483292:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788971870.483296:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000100:00000010:2.0:1788971870.483299:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e120. 00000800:00000200:3.0:1788971870.483300:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971870.483302:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971870.483307:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971870.483308:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800. 00000400:00000200:2.0:1788971870.483311:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971870.483313:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10. 00000400:00000010:2.0:1788971870.483315:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971870.483318:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568256, portal 10 00000400:00000010:2.0:1788971870.483322:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788971870.483324:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189568256, offset 0 00000400:00000200:2.0:1788971870.483328:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971870.483331:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971870.483334:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971870.483336:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971870.483338:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971870.483340:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971870.483342:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971870.483345:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [431:431:254:4294967295] 192.168.203.150@tcp LPNI seq info [431:431:6:4294967295] 00000400:00000200:2.0:1788971870.483348:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971870.483353:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971870.483357:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000200:2.0:1788971870.483357:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971870.483360:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00. 00000800:00000010:3.0:1788971870.483364:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f900. 00000800:00000200:2.0:1788971870.483367:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:3.0:1788971870.483368:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1788971870.483371:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:3.0:1788971870.483372:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000800:00000200:2.0:1788971870.483374:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971870.483376:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:3.0:1788971870.483378:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000400:00000200:2.0:1788971870.483378:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:3.0:1788971870.483387:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ca80 x1875872189568000/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971870.483653:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971870.483656:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:3.0:1788971870.483659:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971870.483663:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971870.483666:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:3.0:1788971870.483668:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788971870.483673:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f480 x1875872189568128/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971870.483743:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971870.483746:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00. 00000400:00000200:3.0:1788971870.483749:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971870.483752:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971870.483754:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:3.0:1788971870.483756:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:3.0:1788971870.484261:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189568256/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971870.490157:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.490167:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971870.490175:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.490180:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.490185:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.490189:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d480 00000400:00000200:3.0:1788971870.490194:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.490200:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x154d [1] + 224 00000400:00000200:3.0:1788971870.490227:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971870.490232:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971870.490246:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f480 x1875872189568128/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971870.490619:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68. 00000400:00000200:2.0:1788971870.490626:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873550 00000400:00000010:2.0:1788971870.490628:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873550. 00000100:00000200:2.0:1788971870.490635:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f480 x1875872189568128/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971870.490645:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f480 x1875872189568128/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971870.490676:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000. 02000000:00000010:2.0:1788971870.490682:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518db00. 00000100:00000010:2.0:1788971870.490686:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480. 00000400:00000200:3.0:1788971870.493184:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.493195:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971870.493203:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.493207:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.493212:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.493215:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780d400 00000400:00000200:3.0:1788971870.493260:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.493266:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x153d [1] + 224 00000400:00000200:3.0:1788971870.493281:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971870.493286:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971870.493299:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ca80 x1875872189568000/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971870.493657:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971870.493663:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:2.0:1788971870.493665:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788971870.493672:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ca80 x1875872189568000/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971870.493682:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ca80 x1875872189568000/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971870.493710:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:2.0:1788971870.493718:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d500. 00000100:00000010:2.0:1788971870.493722:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80. 00000400:00000200:3.0:1788971870.497672:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.497683:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971870.497692:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.497696:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.497701:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.497706:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d500 00000400:00000200:3.0:1788971870.497751:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971870.497768:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x155d [1] + 224 00000400:00000200:3.0:1788971870.497782:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971870.497787:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971870.497801:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189568256/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971870.497858:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10. 00000400:00000200:3.0:1788971870.497869:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:3.0:1788971870.497871:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:3.0:1788971870.497877:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189568256/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971870.497886:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189568256/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971886 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971870.497918:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800. 02000000:00000010:3.0:1788971870.497925:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d200. 00000100:00000010:3.0:1788971870.497930:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000020:00000010:0.0:1788971870.514934:0:112470:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971870.515026:0:112470:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000100:00000010:2.0:1788971871.507206:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ef00. 00000100:00000010:3.0:1788971871.507725:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e120. 00000020:00000010:3.0:1788971871.572314:0:112474:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971871.572410:0:112474:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000020:00000010:1.0:1788971872.615734:0:112478:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971872.616544:0:112478:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:2.0:1788971873.647361:0:112482:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e820. 00000020:00000010:2.0:1788971873.647443:0:112482:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e820. 00000020:00000010:1.0:1788971874.692735:0:112486:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971874.692860:0:112486:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000100:00000010:0.0:1788971875.602726:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788971875.602758:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be200. 02000000:00000010:1.0:1788971875.602862:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4a00. 00000400:00000200:1.0:1788971875.602874:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971875.602880:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971875.602888:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971875.602898:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568384, portal 25 00000400:00000010:1.0:1788971875.602904:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788971875.602909:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189568384, offset 0 00000400:00000200:1.0:1788971875.602920:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971875.602931:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971875.602941:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971875.602948:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971875.602952:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971875.602957:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971875.602960:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971875.602964:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [432:432:256:4294967295] 192.168.203.150@tcp LPNI seq info [432:432:8:4294967295] 00000400:00000200:1.0:1788971875.602969:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971875.602979:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971875.602987:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971875.602992:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f600. 00000800:00000200:1.0:1788971875.603000:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971875.603009:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971875.603015:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971875.603038:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971875.603046:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467600. 00000100:00000010:1.0:1788971875.603058:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467d20. 00000100:00000010:0.0:1788971875.604040:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788971875.604055:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be800. 00000800:00000200:3.0:1788971875.604143:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971875.604149:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f600. 00000400:00000200:3.0:1788971875.604156:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971875.604162:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971875.604169:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525088 00000400:00000010:3.0:1788971875.604182:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525088. 00000100:00000200:3.0:1788971875.604193:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189568384/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971875.607282:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.607295:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971875.607305:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.607311:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.607317:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.607322:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780d580 00000400:00000200:3.0:1788971875.607446:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.607454:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x156d [1] + 224 00000400:00000200:3.0:1788971875.607472:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971875.607477:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971875.607494:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189568384/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788971875.608107:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800. 02000000:00000010:0.0:1788971875.608128:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be400. 00000400:00000010:1.0:1788971875.617549:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971875.617572:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:1.0:1788971875.617575:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971875.617596:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189568384/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971875.617641:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189568384/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971875.617680:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4a00. 02000000:00000010:1.0:1788971875.617688:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be200. 00000100:00000010:1.0:1788971875.617693:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 02000000:00000010:1.0:1788971875.617731:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4a00. 00000400:00000200:1.0:1788971875.617740:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971875.617745:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971875.617752:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971875.617760:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568640, portal 10 00000400:00000010:1.0:1788971875.617769:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788971875.617774:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189568640, offset 0 00000400:00000200:1.0:1788971875.617783:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971875.617793:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971875.617802:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971875.617809:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971875.617840:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971875.617846:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971875.617849:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971875.617853:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [433:433:256:4294967295] 192.168.203.150@tcp LPNI seq info [433:433:8:4294967295] 00000400:00000200:1.0:1788971875.617858:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971875.617870:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971875.617876:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971875.617882:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0ff00. 00000800:00000200:1.0:1788971875.617889:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971875.617898:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971875.617904:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971875.617934:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788971875.617955:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4400. 00000400:00000200:1.0:1788971875.617965:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971875.617968:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000010:1.0:1788971875.617972:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788971875.617974:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568512, portal 10 00000400:00000010:1.0:1788971875.617980:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788971875.617982:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189568512, offset 0 00000400:00000200:1.0:1788971875.617986:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971875.617990:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971875.617994:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971875.617997:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971875.617999:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971875.618002:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971875.618004:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971875.618007:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [434:434:255:4294967295] 192.168.203.150@tcp LPNI seq info [434:434:7:4294967295] 00000400:00000200:1.0:1788971875.618010:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971875.618016:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971875.618020:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971875.618025:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f600. 00000800:00000200:1.0:1788971875.618029:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971875.618033:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971875.618036:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971875.618041:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971875.618997:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971875.619005:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0ff00. 00000400:00000200:3.0:1788971875.619012:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971875.619020:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971875.619027:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:3.0:1788971875.619030:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:3.0:1788971875.619040:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b800 x1875872189568640/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971875.619125:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971875.619128:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f600. 00000400:00000200:3.0:1788971875.619130:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971875.619133:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971875.619136:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:3.0:1788971875.619137:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:3.0:1788971875.619142:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88000 x1875872189568512/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971875.629182:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.629202:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971875.629214:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.629219:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.629225:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.629230:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d680 00000400:00000200:3.0:1788971875.629236:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.629357:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1575 [1] + 224 00000400:00000200:3.0:1788971875.629377:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971875.629382:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971875.629401:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b800 x1875872189568640/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971875.630129:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.630152:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971875.630158:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.630161:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.630165:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.630168:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d600 00000400:00000200:3.0:1788971875.630173:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971875.630177:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x157d [1] + 224 00000400:00000200:3.0:1788971875.630262:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971875.630267:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971875.630276:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88000 x1875872189568512/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971875.630372:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000200:1.0:1788971875.630380:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68 00000400:00000010:1.0:1788971875.630383:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788971875.630390:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88000 x1875872189568512/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971875.630400:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88000 x1875872189568512/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971875.630436:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971875.630446:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:1.0:1788971875.630448:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971875.630454:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b800 x1875872189568640/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971875.630462:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b800 x1875872189568640/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971891 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971875.630480:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4400. 02000000:00000010:1.0:1788971875.630487:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be800. 00000100:00000010:1.0:1788971875.630493:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88000. 02000000:00000010:1.0:1788971875.630498:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4a00. 02000000:00000010:1.0:1788971875.630500:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be400. 00000100:00000010:1.0:1788971875.630503:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800. 00000020:00000010:0.0:1788971875.747508:0:112491:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47801d0. 00000020:00000010:0.0:1788971875.747653:0:112491:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47801d0. 00000100:00000010:3.0:1788971876.626100:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467d20. 00000100:00000010:2.0:1788971876.626374:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467600. 00000020:00000010:1.0:1788971876.807713:0:112495:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971876.807848:0:112495:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:2.0:1788971877.868440:0:112499:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e820. 00000020:00000010:2.0:1788971877.868557:0:112499:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e820. 00000020:00000010:0.0:1788971878.916382:0:112503:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971878.916525:0:112503:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000020:00000010:2.0:1788971879.987612:0:112507:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e820. 00000020:00000010:2.0:1788971879.987752:0:112507:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e820. 00000100:00000010:3.0:1788971880.722115:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788971880.722144:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788971880.722183:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791880. 02000000:00000010:3.0:1788971880.722187:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788971880.722192:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791180. 02000000:00000010:3.0:1788971880.722196:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200. 02000000:00000010:3.0:1788971880.722236:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c9447200. 00000400:00000200:3.0:1788971880.722257:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971880.722262:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788971880.723146:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788971880.723162:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568896, portal 10 00000400:00000010:3.0:1788971880.723173:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873000. 00000100:00000200:3.0:1788971880.723176:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189568896, offset 0 00000400:00000200:3.0:1788971880.723188:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971880.723201:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.723210:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971880.723217:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971880.723221:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971880.723226:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1788971880.723234:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788971880.723243:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.723251:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000010:2.0:1788971880.723252:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000200:3.0:1788971880.723258:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [435:435:256:4294967295] 192.168.203.150@tcp LPNI seq info [435:435:8:4294967295] 00000400:00000010:2.0:1788971880.723260:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000400:00000200:3.0:1788971880.723266:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:2.0:1788971880.723267:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189568768, portal 25 00000400:00000010:2.0:1788971880.723273:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788971880.723276:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189568768, offset 0 00000400:00000200:2.0:1788971880.723284:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971880.723290:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971880.723302:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971880.723308:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788971880.723323:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971880.723333:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971880.723339:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971880.723356:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971880.723362:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09420. 00000100:00000010:3.0:1788971880.723373:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09780. 00000800:00000200:3.0:1788971880.723532:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971880.723536:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:3.0:1788971880.723541:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971880.723546:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971880.729700:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971880.729725:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971880.729732:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971880.729745:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971880.729750:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971880.729753:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971880.729757:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [436:436:255:4294967295] 192.168.203.150@tcp LPNI seq info [436:436:7:4294967295] 00000400:00000200:2.0:1788971880.729766:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971880.729778:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971880.729791:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971880.729808:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f100. 00000800:00000200:2.0:1788971880.729837:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971880.729851:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971880.729860:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971880.729870:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971880.729926:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b800. 00000400:00000200:2.0:1788971880.729936:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971880.729941:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971880.729952:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971880.729961:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569024, portal 10 00000400:00000010:2.0:1788971880.729971:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788971880.730011:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189569024, offset 0 00000400:00000200:2.0:1788971880.730022:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971880.730034:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971880.730038:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971880.730041:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971880.730043:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971880.730050:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971880.730052:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971880.730055:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [437:437:254:4294967295] 192.168.203.150@tcp LPNI seq info [437:437:6:4294967295] 00000400:00000200:2.0:1788971880.730062:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971880.730067:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971880.730073:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971880.730076:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f400. 00000800:00000200:2.0:1788971880.730081:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971880.730084:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971880.730087:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971880.730098:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.731174:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873000 00000400:00000010:3.0:1788971880.731181:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873000. 00000100:00000200:3.0:1788971880.731198:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791880 x1875872189568896/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971880.731247:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.731253:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971880.731259:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.731263:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.731277:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.731282:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d780 00000400:00000200:3.0:1788971880.731286:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.731293:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1585 [1] + 224 00000400:00000200:3.0:1788971880.731310:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971880.731315:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971880.731337:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791880 x1875872189568896/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971880.731401:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971880.731405:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f100. 00000400:00000200:3.0:1788971880.731409:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971880.731413:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971880.731415:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:3.0:1788971880.731417:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:3.0:1788971880.731423:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790e00 x1875872189568768/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971880.731512:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971880.731515:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f400. 00000400:00000200:3.0:1788971880.731519:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971880.731522:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971880.731525:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:3.0:1788971880.731527:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:3.0:1788971880.731532:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791180 x1875872189569024/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971880.736897:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:2.0:1788971880.736918:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:2.0:1788971880.736921:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:2.0:1788971880.736940:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791880 x1875872189568896/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971880.736953:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791880 x1875872189568896/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971880.736998:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c9447200. 02000000:00000010:2.0:1788971880.737008:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788971880.737013:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791880. 00000400:00000200:3.0:1788971880.740728:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741007:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971880.741017:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741023:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741028:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741032:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780d700 00000400:00000200:3.0:1788971880.741038:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741045:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1595 [1] + 224 00000400:00000200:3.0:1788971880.741058:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971880.741063:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971880.741077:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790e00 x1875872189568768/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971880.741204:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741208:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971880.741213:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741215:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741218:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741221:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d800 00000400:00000200:3.0:1788971880.741225:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971880.741229:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x15a5 [1] + 224 00000400:00000200:3.0:1788971880.741292:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971880.741298:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971880.741305:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791180 x1875872189569024/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971880.741432:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971880.741438:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:2.0:1788971880.741440:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788971880.741456:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790e00 x1875872189568768/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971880.741465:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790e00 x1875872189568768/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971880.741692:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:2.0:1788971880.741696:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788971880.741698:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971880.741703:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791180 x1875872189569024/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971880.741712:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791180 x1875872189569024/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971896 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971880.741732:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:2.0:1788971880.742077:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788971880.742083:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790e00. 02000000:00000010:2.0:1788971880.742088:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b800. 02000000:00000010:2.0:1788971880.742092:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200. 00000100:00000010:2.0:1788971880.742094:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791180. 00000020:00000010:3.0:1788971881.030099:0:112511:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971881.030174:0:112511:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000100:00000010:3.0:1788971881.745993:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09780. 00000100:00000010:2.0:1788971881.746019:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09420. 00000020:00000010:0.0:1788971882.075857:0:112515:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971882.075955:0:112515:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000020:00000010:1.0:1788971883.116093:0:112520:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0090. 00000020:00000010:1.0:1788971883.116182:0:112520:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0090. 00000020:00000010:0.0:1788971884.148512:0:112524:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971884.148610:0:112524:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000020:00000010:2.0:1788971885.180111:0:112528:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e820. 00000020:00000010:2.0:1788971885.180206:0:112528:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e820. 00000100:00000010:3.0:1788971885.841904:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788971885.841929:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788971885.841961:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790700. 02000000:00000010:3.0:1788971885.841966:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788971885.841972:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788971885.841978:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67200. 02000000:00000010:2.0:1788971885.842365:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788971885.842376:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971885.842381:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971885.842388:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971885.842395:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569152, portal 25 00000400:00000010:2.0:1788971885.842400:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788971885.842403:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189569152, offset 0 00000400:00000200:2.0:1788971885.842412:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971885.842422:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971885.842429:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971885.842447:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971885.842451:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971885.842455:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971885.842459:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971885.842462:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [438:438:256:4294967295] 192.168.203.150@tcp LPNI seq info [438:438:8:4294967295] 00000400:00000200:2.0:1788971885.842481:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971885.842491:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971885.842496:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971885.842502:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000. 00000800:00000200:2.0:1788971885.842509:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971885.842516:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971885.842520:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971885.842540:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971885.842545:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e900. 00000100:00000010:2.0:1788971885.842555:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651eba0. 02000000:00000010:2.0:1788971885.842566:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000400:00000200:2.0:1788971885.842570:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971885.842573:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971885.842576:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971885.842579:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569408, portal 10 00000400:00000010:2.0:1788971885.842586:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788971885.842587:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189569408, offset 0 00000400:00000200:2.0:1788971885.842592:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971885.842596:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971885.842599:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971885.842601:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971885.842604:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971885.842606:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971885.842607:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971885.842610:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [439:439:255:4294967295] 192.168.203.150@tcp LPNI seq info [439:439:7:4294967295] 00000400:00000200:2.0:1788971885.842613:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971885.842625:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971885.842629:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971885.842632:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f500. 00000800:00000200:2.0:1788971885.842635:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971885.842640:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971885.842642:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971885.842648:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971885.842669:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a400. 00000400:00000200:2.0:1788971885.842674:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971885.842676:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126aa8. 00000400:00000010:2.0:1788971885.842680:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788971885.842683:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569280, portal 10 00000400:00000010:2.0:1788971885.842688:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788971885.842690:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189569280, offset 0 00000400:00000200:2.0:1788971885.842693:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971885.842697:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971885.842701:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971885.842703:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971885.842706:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971885.842707:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971885.842709:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971885.842711:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [440:440:254:4294967295] 192.168.203.150@tcp LPNI seq info [440:440:6:4294967295] 00000400:00000200:2.0:1788971885.842715:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971885.842719:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971885.842723:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971885.842726:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f400. 00000800:00000200:2.0:1788971885.842729:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971885.842733:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971885.842735:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971885.842740:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971885.843317:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971885.843323:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000. 00000400:00000200:3.0:1788971885.843330:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971885.843335:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971885.843341:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:3.0:1788971885.843343:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:3.0:1788971885.843353:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189569152/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971885.843392:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971885.843394:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f500. 00000400:00000200:3.0:1788971885.843396:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971885.843400:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971885.843402:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:3.0:1788971885.843403:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:3.0:1788971885.843407:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792680 x1875872189569408/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971885.843488:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971885.843491:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f400. 00000400:00000200:3.0:1788971885.843495:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971885.843498:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971885.843500:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:3.0:1788971885.843514:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:3.0:1788971885.843520:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790700 x1875872189569280/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971885.845900:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.845910:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971885.845918:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.845923:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.845928:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.845932:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d900 00000400:00000200:3.0:1788971885.845938:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.845944:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x15d5 [1] + 224 00000400:00000200:3.0:1788971885.845955:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971885.845960:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971885.845973:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790700 x1875872189569280/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971885.846012:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.846028:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971885.846033:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.846035:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.846038:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.846041:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780d880 00000400:00000200:3.0:1788971885.846045:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.846049:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x15b5 [1] + 224 00000400:00000200:3.0:1788971885.846062:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971885.846065:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971885.846071:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189569152/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971885.847017:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126aa8. 00000400:00000200:2.0:1788971885.847025:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b5d8 00000400:00000010:2.0:1788971885.847028:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788971885.847037:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790700 x1875872189569280/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971885.847047:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790700 x1875872189569280/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971885.847073:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:2.0:1788971885.847075:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788971885.847077:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971885.847081:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189569152/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971885.847088:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189569152/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971885.847100:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a400. 02000000:00000010:2.0:1788971885.847106:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788971885.847110:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790700. 02000000:00000010:2.0:1788971885.847113:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:2.0:1788971885.847115:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:2.0:1788971885.847117:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 00000400:00000200:3.0:1788971885.857133:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.857143:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971885.857152:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.857157:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.857163:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.857167:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780d980 00000400:00000200:3.0:1788971885.857173:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971885.857179:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x15c5 [1] + 224 00000400:00000200:3.0:1788971885.857235:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971885.857241:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971885.857256:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792680 x1875872189569408/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971885.857547:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971885.857553:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971885.857556:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971885.857564:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792680 x1875872189569408/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971885.857575:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792680 x1875872189569408/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971901 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971885.857610:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:2.0:1788971885.857620:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67200. 00000100:00000010:2.0:1788971885.857624:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792680. 00000020:00000010:3.0:1788971886.220575:0:112532:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971886.220661:0:112532:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000100:00000010:3.0:1788971886.865961:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651eba0. 00000100:00000010:2.0:1788971886.866578:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e900. 00000020:00000010:0.0:1788971887.259391:0:112536:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971887.259496:0:112536:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971888.297899:0:112540:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971888.297998:0:112540:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000020:00000010:3.0:1788971889.345013:0:112544:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971889.345134:0:112544:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000020:00000010:0.0:1788971890.397712:0:112549:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971890.397862:0:112549:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000100:00000010:2.0:1788971890.962363:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80. 02000000:00000010:2.0:1788971890.962404:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100. 00000100:00000010:2.0:1788971890.962455:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788971890.962460:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060ff00. 00000100:00000010:2.0:1788971890.962470:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971890.962474:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f900. 02000000:00000010:2.0:1788971890.962509:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788971890.962521:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971890.962526:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971890.962534:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971890.962543:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569664, portal 10 00000400:00000010:2.0:1788971890.962563:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788971890.962567:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189569664, offset 0 00000400:00000200:2.0:1788971890.962578:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971890.962588:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971890.962597:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971890.962604:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971890.962608:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971890.962613:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971890.962616:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971890.962620:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [441:441:256:4294967295] 192.168.203.150@tcp LPNI seq info [441:441:8:4294967295] 00000400:00000200:2.0:1788971890.962625:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971890.962636:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971890.962643:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971890.962656:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f400. 00000800:00000200:2.0:1788971890.962663:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971890.962672:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971890.962679:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971890.962693:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971890.962699:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e900. 00000100:00000010:2.0:1788971890.962728:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e540. 02000000:00000010:2.0:1788971890.962759:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a400. 00000400:00000200:2.0:1788971890.962764:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971890.962767:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971890.962770:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788971890.962773:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569536, portal 25 00000400:00000010:2.0:1788971890.962777:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788971890.962779:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189569536, offset 0 00000400:00000200:2.0:1788971890.962783:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971890.962787:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971890.962790:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971890.962793:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971890.962795:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971890.962797:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971890.962800:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971890.962802:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [442:442:255:4294967295] 192.168.203.150@tcp LPNI seq info [442:442:7:4294967295] 00000400:00000200:2.0:1788971890.962806:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971890.964793:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971890.964809:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971890.964843:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f500. 00000800:00000200:2.0:1788971890.964852:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971890.964860:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971890.964865:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971890.964877:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971890.964911:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b000. 00000400:00000200:2.0:1788971890.964917:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971890.964922:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126aa8. 00000400:00000010:2.0:1788971890.964928:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bdd0. 00000100:00000200:2.0:1788971890.964935:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569792, portal 10 00000400:00000010:2.0:1788971890.964943:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788971890.964946:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189569792, offset 0 00000400:00000200:2.0:1788971890.964974:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971890.964983:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971890.964989:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971890.964994:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971890.964999:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971890.965003:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971890.965005:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971890.965009:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [443:443:254:4294967295] 192.168.203.150@tcp LPNI seq info [443:443:6:4294967295] 00000400:00000200:2.0:1788971890.965013:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971890.965020:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971890.965025:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971890.965028:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000. 00000800:00000200:2.0:1788971890.965032:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971890.965036:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971890.965038:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971890.965044:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971890.965352:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971890.965374:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f400. 00000400:00000200:3.0:1788971890.965382:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971890.965388:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971890.965395:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:3.0:1788971890.965398:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:3.0:1788971890.965411:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f480 x1875872189569664/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971890.965448:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971890.965451:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f500. 00000400:00000200:3.0:1788971890.965454:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971890.965458:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971890.965460:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:3.0:1788971890.965462:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:3.0:1788971890.965468:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ca80 x1875872189569536/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971890.965555:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971890.965558:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000. 00000400:00000200:3.0:1788971890.965561:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971890.965564:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971890.965567:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:3.0:1788971890.965569:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:3.0:1788971890.965574:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189569792/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971890.967182:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967192:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971890.967200:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967205:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967210:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967214:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780da80 00000400:00000200:3.0:1788971890.967220:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967227:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x15e5 [1] + 224 00000400:00000200:3.0:1788971890.967252:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971890.967257:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971890.967270:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f480 x1875872189569664/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971890.967305:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967308:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971890.967311:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967312:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967314:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967316:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780db00 00000400:00000200:3.0:1788971890.967318:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.967321:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1605 [1] + 224 00000400:00000200:3.0:1788971890.967327:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971890.967330:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971890.967334:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189569792/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971890.967529:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126aa8. 00000400:00000200:2.0:1788971890.967536:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bdd0 00000400:00000010:2.0:1788971890.967538:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bdd0. 00000100:00000200:2.0:1788971890.967545:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189569792/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971890.967927:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189569792/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971890.967962:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971890.967966:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971890.967968:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971890.967972:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f480 x1875872189569664/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971890.967980:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f480 x1875872189569664/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971890.968019:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b000. 02000000:00000010:2.0:1788971890.968027:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f900. 00000100:00000010:2.0:1788971890.968032:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971890.968036:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:2.0:1788971890.968040:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060ff00. 00000100:00000010:2.0:1788971890.968042:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480. 00000400:00000200:3.0:1788971890.968619:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.968626:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971890.968632:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.968636:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.968640:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.968645:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780da00 00000400:00000200:3.0:1788971890.968660:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971890.968667:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x15f5 [1] + 224 00000400:00000200:3.0:1788971890.968739:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971890.968745:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971890.968755:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ca80 x1875872189569536/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971890.969174:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:2.0:1788971890.969181:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b5d8 00000400:00000010:2.0:1788971890.969184:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788971890.969191:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ca80 x1875872189569536/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971890.969201:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ca80 x1875872189569536/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971906 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971890.969224:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a400. 02000000:00000010:2.0:1788971890.969231:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100. 00000100:00000010:2.0:1788971890.969235:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80. 00000020:00000010:0.0:1788971891.471194:0:112553:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971891.471301:0:112553:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000100:00000010:2.0:1788971891.985966:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e900. 00000100:00000010:3.0:1788971891.986119:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e540. 00000020:00000010:0.0:1788971892.514550:0:112557:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780b70. 00000020:00000010:0.0:1788971892.515900:0:112557:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780b70. 00000020:00000010:3.0:1788971893.546451:0:112561:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971893.546574:0:112561:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971893.589863:0:112562:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5970. 00000020:00000010:3.0:1788971893.589976:0:112562:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5970. 00000100:00000010:3.0:1788971896.082237:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788971896.082271:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 02000000:00000010:2.0:1788971896.082338:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b200. 00000400:00000200:2.0:1788971896.082352:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971896.082358:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971896.082366:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788971896.082375:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189569920, portal 25 00000400:00000010:2.0:1788971896.082393:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971896.082404:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189569920, offset 0 00000400:00000200:2.0:1788971896.082416:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971896.082427:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.082438:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971896.082445:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971896.082450:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971896.082455:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971896.082478:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971896.082483:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [444:444:256:4294967295] 192.168.203.150@tcp LPNI seq info [444:444:8:4294967295] 00000400:00000200:2.0:1788971896.082489:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.082501:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971896.082508:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971896.082514:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f100. 00000800:00000200:2.0:1788971896.082522:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971896.082530:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971896.082536:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971896.082566:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971896.082572:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e900. 00000100:00000010:2.0:1788971896.082586:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e960. 00000800:00000200:2.0:1788971896.082991:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971896.082997:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f100. 00000400:00000200:2.0:1788971896.083003:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971896.083009:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971896.083016:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971896.083018:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971896.083028:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189569920/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788971896.084498:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788971896.084523:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788971896.084563:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791f80. 02000000:00000010:3.0:1788971896.084569:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300. 02000000:00000010:3.0:1788971896.084612:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2e00. 00000400:00000200:3.0:1788971896.084621:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971896.084626:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788971896.084635:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873000. 00000100:00000200:3.0:1788971896.084641:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570176, portal 10 00000400:00000010:3.0:1788971896.084649:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971896.084653:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189570176, offset 0 00000400:00000200:3.0:1788971896.084663:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971896.084672:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.084680:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971896.084686:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971896.084690:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971896.084694:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971896.084696:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971896.084700:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [445:445:256:4294967295] 192.168.203.150@tcp LPNI seq info [445:445:8:4294967295] 00000400:00000200:3.0:1788971896.084704:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.084714:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971896.084721:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971896.084726:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788971896.084735:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 02000000:00000010:2.0:1788971896.084737:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000800:00000200:3.0:1788971896.084744:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:2.0:1788971896.084746:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971896.084750:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971896.084754:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971896.084757:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570048, portal 10 00000400:00000010:2.0:1788971896.084763:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bdd0. 00000100:00000200:2.0:1788971896.084767:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189570048, offset 0 00000800:00000200:3.0:1788971896.084768:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971896.084776:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971896.084782:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.084789:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971896.084790:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.084795:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971896.084807:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971896.084810:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971896.084831:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971896.084835:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [446:446:255:4294967295] 192.168.203.150@tcp LPNI seq info [446:446:7:4294967295] 00000400:00000200:2.0:1788971896.084840:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.084846:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971896.084850:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971896.084854:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f100. 00000800:00000200:2.0:1788971896.084859:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971896.084863:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000200:2.0:1788971896.084864:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000010:3.0:1788971896.084869:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000800:00000200:2.0:1788971896.084870:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971896.084877:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971896.084881:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.084887:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971896.084895:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:3.0:1788971896.084898:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971896.084907:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791f80 x1875872189570176/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971896.084972:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971896.084975:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f100. 00000400:00000200:3.0:1788971896.084978:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971896.084981:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971896.084984:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bdd0 00000400:00000010:3.0:1788971896.084985:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bdd0. 00000100:00000200:3.0:1788971896.084990:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790380 x1875872189570048/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971896.088638:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.088650:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971896.088661:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.088667:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.088673:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.088678:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780db80 00000400:00000200:2.0:1788971896.088694:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971896.088703:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1615 [1] + 224 00000400:00000200:2.0:1788971896.088725:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971896.088730:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971896.088747:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189569920/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971896.088901:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:3.0:1788971896.088908:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b5d8 00000400:00000010:3.0:1788971896.088912:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b5d8. 00000100:00000200:3.0:1788971896.088919:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189569920/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971896.088938:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189569920/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971896.088980:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b200. 02000000:00000010:3.0:1788971896.088991:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788971896.088997:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 00000400:00000200:3.0:1788971896.090683:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090703:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971896.090714:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090739:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090745:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090749:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780dc80 00000400:00000200:3.0:1788971896.090756:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090763:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1625 [1] + 224 00000400:00000200:3.0:1788971896.090776:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971896.090782:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971896.090796:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791f80 x1875872189570176/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971896.090876:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090880:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971896.090885:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090887:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090890:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090893:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780dc00 00000400:00000200:3.0:1788971896.090896:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971896.090900:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1635 [1] + 224 00000400:00000200:3.0:1788971896.090976:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971896.090980:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971896.090986:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790380 x1875872189570048/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971896.091341:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971896.091349:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971896.091352:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971896.091360:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790380 x1875872189570048/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971896.091370:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790380 x1875872189570048/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971896.091429:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:2.0:1788971896.091440:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788971896.091446:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380. 00000400:00000010:2.0:1788971896.091473:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:2.0:1788971896.091477:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873000 00000400:00000010:2.0:1788971896.091479:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873000. 00000100:00000200:2.0:1788971896.091486:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791f80 x1875872189570176/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971896.091495:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791f80 x1875872189570176/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971912 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971896.091509:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2e00. 02000000:00000010:2.0:1788971896.091513:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300. 00000100:00000010:2.0:1788971896.091516:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791f80. 00000100:00000010:2.0:1788971897.105970:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e900. 00000100:00000010:3.0:1788971897.110173:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e960. 00000100:00000010:2.0:1788971901.202474:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80. 02000000:00000010:2.0:1788971901.202505:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060ff00. 02000000:00000010:3.0:1788971901.202593:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2e00. 00000400:00000200:3.0:1788971901.202606:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971901.202613:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788971901.202621:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971901.202630:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570304, portal 25 00000400:00000010:3.0:1788971901.202636:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788971901.202640:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189570304, offset 0 00000400:00000200:3.0:1788971901.202653:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971901.202693:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.202705:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971901.202720:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971901.202726:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971901.202731:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971901.202734:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971901.202739:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [447:447:256:4294967295] 192.168.203.150@tcp LPNI seq info [447:447:8:4294967295] 00000400:00000200:3.0:1788971901.202743:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.202753:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971901.202759:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971901.202766:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788971901.202774:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971901.202783:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971901.202789:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971901.202807:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971901.202832:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09780. 00000100:00000010:3.0:1788971901.202846:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09b40. 00000100:00000010:2.0:1788971901.202975:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788971901.202984:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f900. 02000000:00000010:3.0:1788971901.203466:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2800. 00000400:00000200:3.0:1788971901.203474:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971901.203478:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788971901.203484:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788971901.203489:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570432, portal 10 00000400:00000010:3.0:1788971901.203497:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971901.203499:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189570432, offset 0 00000400:00000200:3.0:1788971901.203505:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971901.203511:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.203515:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971901.203518:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971901.203520:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971901.203523:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971901.203526:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971901.203538:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [448:448:255:4294967295] 192.168.203.150@tcp LPNI seq info [448:448:7:4294967295] 00000400:00000200:3.0:1788971901.203542:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.203601:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971901.203608:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971901.203614:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788971901.203620:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971901.203626:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971901.203629:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971901.203637:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971901.203786:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971901.203790:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:3.0:1788971901.203799:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971901.203804:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971901.203810:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28 00000400:00000010:3.0:1788971901.203971:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788971901.203997:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ca80 x1875872189570304/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788971901.204204:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971901.204219:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100. 02000000:00000010:2.0:1788971901.204267:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000400:00000200:2.0:1788971901.204274:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971901.204277:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971901.204282:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971901.204286:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570560, portal 10 00000400:00000010:2.0:1788971901.204292:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b330. 00000100:00000200:2.0:1788971901.204294:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189570560, offset 0 00000400:00000200:2.0:1788971901.204301:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971901.204306:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971901.204310:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971901.204323:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971901.204326:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971901.204328:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971901.204331:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971901.204334:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [449:449:255:4294967295] 192.168.203.150@tcp LPNI seq info [449:449:7:4294967295] 00000400:00000200:2.0:1788971901.204339:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971901.204348:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971901.204353:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971901.204357:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000. 00000800:00000200:2.0:1788971901.204370:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971901.204376:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971901.204379:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971901.204386:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971901.208394:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971901.208470:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:3.0:1788971901.208476:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971901.208484:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971901.208491:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788971901.208494:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971901.208505:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f480 x1875872189570432/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971901.208542:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.208547:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971901.208553:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.208557:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.208562:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.208578:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780dd00 00000400:00000200:3.0:1788971901.208583:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.208588:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1645 [1] + 224 00000400:00000200:3.0:1788971901.208607:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971901.208611:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971901.208619:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ca80 x1875872189570304/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971901.211235:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971901.211243:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000. 00000400:00000200:3.0:1788971901.211249:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971901.211257:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971901.211263:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b330 00000400:00000010:3.0:1788971901.211266:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b330. 00000100:00000200:3.0:1788971901.211277:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189570560/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971901.211309:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.211314:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971901.211320:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.211324:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.211328:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.211332:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780dd80 00000400:00000200:3.0:1788971901.211337:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.211343:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1655 [1] + 224 00000400:00000200:3.0:1788971901.211355:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971901.211367:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971901.211374:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f480 x1875872189570432/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971901.212068:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:3.0:1788971901.212074:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770 00000400:00000010:3.0:1788971901.212076:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788971901.212097:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f480 x1875872189570432/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971901.212107:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f480 x1875872189570432/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971901.212134:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788971901.212136:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:3.0:1788971901.212138:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971901.212143:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ca80 x1875872189570304/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971901.212152:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ca80 x1875872189570304/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971901.212165:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2800. 02000000:00000010:3.0:1788971901.212171:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f900. 00000100:00000010:3.0:1788971901.212175:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480. 02000000:00000010:3.0:1788971901.212179:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2e00. 02000000:00000010:3.0:1788971901.212182:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060ff00. 00000100:00000010:3.0:1788971901.212184:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80. 00000400:00000200:3.0:1788971901.215067:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.215076:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971901.215085:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.215090:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.215095:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.215099:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780de00 00000400:00000200:3.0:1788971901.215105:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971901.215110:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1665 [1] + 224 00000400:00000200:3.0:1788971901.215123:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971901.215128:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971901.215140:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189570560/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971901.215302:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:2.0:1788971901.215307:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:2.0:1788971901.215309:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971901.215315:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189570560/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971901.215325:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189570560/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971917 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971901.215355:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:2.0:1788971901.215371:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100. 00000100:00000010:2.0:1788971901.215376:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000100:00000010:3.0:1788971902.225935:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09b40. 00000100:00000010:2.0:1788971902.226152:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09780. 00000100:00000010:2.0:1788971906.323996:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788971906.324022:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100. 00000100:00000010:2.0:1788971906.324055:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788971906.324060:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f500. 00000100:00000010:2.0:1788971906.324066:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f800. 02000000:00000010:2.0:1788971906.324070:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f400. 02000000:00000010:2.0:1788971906.324151:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000400:00000200:2.0:1788971906.324162:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971906.324166:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971906.324173:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788971906.324180:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570688, portal 25 00000400:00000010:2.0:1788971906.324185:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788971906.324188:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189570688, offset 0 00000400:00000200:2.0:1788971906.324209:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971906.324219:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.324228:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971906.324694:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971906.324702:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971906.324706:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971906.324709:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971906.324716:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [450:450:256:4294967295] 192.168.203.150@tcp LPNI seq info [450:450:8:4294967295] 00000400:00000200:2.0:1788971906.324721:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.324732:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971906.324738:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971906.324745:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00. 00000800:00000200:2.0:1788971906.324753:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971906.324761:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971906.324766:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971906.324787:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971906.324794:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e900. 00000100:00000010:2.0:1788971906.324806:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ea20. 02000000:00000010:2.0:1788971906.324859:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b000. 00000400:00000200:2.0:1788971906.324866:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971906.324868:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126aa8. 00000400:00000010:2.0:1788971906.325618:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971906.325628:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570944, portal 10 00000400:00000010:2.0:1788971906.325635:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788971906.325650:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189570944, offset 0 00000400:00000200:2.0:1788971906.325658:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971906.325664:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.325670:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971906.325674:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971906.325677:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971906.325679:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971906.325681:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971906.325685:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [451:451:255:4294967295] 192.168.203.150@tcp LPNI seq info [451:451:7:4294967295] 00000400:00000200:2.0:1788971906.325689:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.325696:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971906.325701:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971906.325706:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f900. 00000800:00000200:2.0:1788971906.325711:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971906.325716:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971906.325720:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971906.325728:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971906.325754:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788971906.325758:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971906.325761:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d68. 00000400:00000010:2.0:1788971906.325765:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971906.325768:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189570816, portal 10 00000400:00000010:2.0:1788971906.325773:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788971906.325775:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189570816, offset 0 00000400:00000200:2.0:1788971906.325779:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971906.325783:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.325785:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971906.325788:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971906.325790:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971906.325792:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971906.325794:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971906.325804:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [452:452:254:4294967295] 192.168.203.150@tcp LPNI seq info [452:452:6:4294967295] 00000400:00000200:2.0:1788971906.325808:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.325841:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971906.325846:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971906.325850:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000. 00000800:00000200:2.0:1788971906.325854:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971906.325858:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971906.325861:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971906.325867:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971906.325921:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971906.325925:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00. 00000400:00000200:2.0:1788971906.325932:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971906.325937:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971906.325942:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:2.0:1788971906.325944:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788971906.326611:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189570688/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971906.326642:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971906.326644:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f900. 00000400:00000200:2.0:1788971906.326648:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971906.326652:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971906.326655:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788971906.326656:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788971906.326662:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f800 x1875872189570944/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971906.326730:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971906.326733:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000. 00000400:00000200:2.0:1788971906.326735:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971906.326739:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971906.326741:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788971906.326743:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788971906.326757:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189570816/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971906.331251:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331261:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971906.331269:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331274:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331280:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331284:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780df80 00000400:00000200:3.0:1788971906.331290:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331296:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1685 [1] + 224 00000400:00000200:3.0:1788971906.331307:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971906.331312:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971906.331336:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f800 x1875872189570944/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971906.331415:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331419:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971906.331424:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331433:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331436:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331438:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780df00 00000400:00000200:3.0:1788971906.331442:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971906.331446:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1695 [1] + 224 00000400:00000200:3.0:1788971906.331455:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971906.331459:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971906.331465:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189570816/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971906.331807:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d68. 00000400:00000200:2.0:1788971906.331840:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:2.0:1788971906.331842:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971906.331850:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189570816/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971906.331858:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189570816/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971906.331900:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126aa8. 00000400:00000200:2.0:1788971906.331903:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788971906.331904:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971906.331908:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f800 x1875872189570944/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971906.331915:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f800 x1875872189570944/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971906.331929:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:2.0:1788971906.331937:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f500. 00000100:00000010:2.0:1788971906.331940:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788971906.331944:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b000. 02000000:00000010:2.0:1788971906.331947:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f400. 00000100:00000010:2.0:1788971906.331949:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f800. 00000400:00000200:2.0:1788971906.332801:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.332847:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971906.332857:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.332863:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.332868:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.332873:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780de80 00000400:00000200:2.0:1788971906.332879:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971906.333025:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1675 [1] + 224 00000400:00000200:2.0:1788971906.333043:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971906.333048:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971906.333064:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189570688/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971906.333168:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:3.0:1788971906.333174:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:3.0:1788971906.333177:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:3.0:1788971906.333192:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189570688/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971906.333202:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189570688/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971922 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971906.333236:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:3.0:1788971906.333244:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100. 00000100:00000010:3.0:1788971906.333249:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000100:00000010:3.0:1788971907.346116:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ea20. 00000100:00000010:2.0:1788971907.346353:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e900. 00000100:00000010:0.0:1788971911.443318:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971911.443348:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09bed00. 00000100:00000010:0.0:1788971911.443391:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788971911.443397:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be300. 00000100:00000010:0.0:1788971911.443403:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788971911.443409:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be400. 02000000:00000010:1.0:1788971911.444132:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5c00. 00000400:00000200:1.0:1788971911.444147:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971911.444153:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971911.444161:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971911.444170:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571072, portal 25 00000400:00000010:1.0:1788971911.444176:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788971911.444181:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189571072, offset 0 00000400:00000200:1.0:1788971911.444193:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971911.444206:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971911.444216:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971911.444224:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971911.444228:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971911.444233:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971911.444237:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971911.444242:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [453:453:256:4294967295] 192.168.203.150@tcp LPNI seq info [453:453:8:4294967295] 00000400:00000200:1.0:1788971911.444271:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971911.444284:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971911.444302:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971911.444309:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0fa00. 00000800:00000200:1.0:1788971911.444317:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971911.444327:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971911.444333:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0fa00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971911.444360:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971911.444366:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74672a0. 00000100:00000010:1.0:1788971911.444381:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467120. 02000000:00000010:1.0:1788971911.444398:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5a00. 00000400:00000200:1.0:1788971911.444402:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971911.444405:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000010:1.0:1788971911.444409:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971911.444413:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571328, portal 10 00000400:00000010:1.0:1788971911.444419:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788971911.444422:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189571328, offset 0 00000400:00000200:1.0:1788971911.444428:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971911.444431:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971911.444435:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971911.444438:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971911.444441:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971911.444443:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971911.444445:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971911.444448:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [454:454:255:4294967295] 192.168.203.150@tcp LPNI seq info [454:454:7:4294967295] 00000400:00000200:1.0:1788971911.444452:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971911.444458:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971911.444463:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971911.444466:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0fb00. 00000800:00000200:1.0:1788971911.444470:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971911.444481:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971911.444484:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971911.444491:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788971911.444515:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4800. 00000400:00000200:1.0:1788971911.444518:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971911.444521:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788971911.444525:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788971911.444528:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571200, portal 10 00000400:00000010:1.0:1788971911.444533:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525cc0. 00000100:00000200:1.0:1788971911.444535:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189571200, offset 0 00000400:00000200:1.0:1788971911.444539:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971911.444543:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971911.444547:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971911.444549:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971911.444552:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971911.444554:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971911.444556:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971911.444559:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [455:455:254:4294967295] 192.168.203.150@tcp LPNI seq info [455:455:6:4294967295] 00000400:00000200:1.0:1788971911.444563:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971911.444569:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971911.444573:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971911.444576:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f800. 00000800:00000200:1.0:1788971911.444580:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971911.444584:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971911.444587:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971911.444591:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971911.444792:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971911.444798:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0fa00. 00000400:00000200:3.0:1788971911.444806:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971911.444811:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971911.444844:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68 00000400:00000010:3.0:1788971911.444847:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68. 00000100:00000200:3.0:1788971911.444860:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88700 x1875872189571072/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971911.444886:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971911.444889:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0fb00. 00000400:00000200:3.0:1788971911.444891:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971911.444894:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971911.444896:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:3.0:1788971911.444899:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:3.0:1788971911.444903:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189571328/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971911.444994:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971911.444997:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f800. 00000400:00000200:3.0:1788971911.444999:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971911.445002:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971911.445005:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525cc0 00000400:00000010:3.0:1788971911.445006:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525cc0. 00000100:00000200:3.0:1788971911.445011:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189571200/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971911.451529:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.451541:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971911.451553:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.451558:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.451563:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.451568:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e000 00000400:00000200:2.0:1788971911.451646:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.451653:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x16a5 [1] + 224 00000400:00000200:2.0:1788971911.451736:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971911.451742:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971911.451757:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88700 x1875872189571072/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971911.452008:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452012:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971911.452017:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452020:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452022:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452025:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e100 00000400:00000200:2.0:1788971911.452028:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452033:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x16ad [1] + 224 00000400:00000200:2.0:1788971911.452038:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971911.452041:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971911.452048:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189571328/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971911.452079:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452082:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971911.452086:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452089:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452091:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452094:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e080 00000400:00000200:2.0:1788971911.452096:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971911.452100:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x16b5 [1] + 224 00000400:00000200:2.0:1788971911.452110:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971911.452113:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971911.452119:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189571200/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971911.462805:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971911.462846:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:1.0:1788971911.462849:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971911.462864:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88700 x1875872189571072/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971911.462898:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88700 x1875872189571072/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971911.462949:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000200:1.0:1788971911.462952:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:1.0:1788971911.462954:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971911.462960:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189571328/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971911.462968:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189571328/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971911.462988:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5c00. 02000000:00000010:1.0:1788971911.462994:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09bed00. 00000100:00000010:1.0:1788971911.462999:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700. 02000000:00000010:1.0:1788971911.463002:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5a00. 02000000:00000010:1.0:1788971911.463004:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be400. 00000100:00000010:1.0:1788971911.463006:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000400:00000010:1.0:1788971911.463012:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971911.463015:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525110 00000400:00000010:1.0:1788971911.463016:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788971911.463022:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189571200/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971911.463030:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189571200/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971927 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971911.463044:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4800. 02000000:00000010:1.0:1788971911.463047:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be300. 00000100:00000010:1.0:1788971911.463049:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 00000100:00000010:2.0:1788971912.465965:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74672a0. 00000100:00000010:3.0:1788971912.466408:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467120. 00000100:00000010:0.0:1788971916.561941:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88e00. 02000000:00000010:0.0:1788971916.561961:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be800. 00000100:00000010:0.0:1788971916.562000:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788971916.562005:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be200. 00000100:00000010:0.0:1788971916.562011:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788971916.562022:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09bef00. 02000000:00000010:1.0:1788971916.562507:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4800. 00000400:00000200:1.0:1788971916.562518:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971916.562522:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788971916.562540:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788971916.562547:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571456, portal 25 00000400:00000010:1.0:1788971916.562552:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971916.562556:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189571456, offset 0 00000400:00000200:1.0:1788971916.562565:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971916.562573:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971916.562581:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971916.562586:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971916.562590:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971916.562594:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971916.562597:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971916.562600:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [456:456:256:4294967295] 192.168.203.150@tcp LPNI seq info [456:456:8:4294967295] 00000400:00000200:1.0:1788971916.562605:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971916.562614:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971916.562620:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971916.562625:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f900. 00000800:00000200:1.0:1788971916.562644:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971916.562651:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971916.562656:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971916.562674:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971916.562678:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467a80. 00000100:00000010:1.0:1788971916.562688:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74677e0. 02000000:00000010:1.0:1788971916.562699:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5a00. 00000400:00000200:1.0:1788971916.562703:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971916.562705:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000010:1.0:1788971916.562709:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971916.562711:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571712, portal 10 00000400:00000010:1.0:1788971916.562716:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525330. 00000100:00000200:1.0:1788971916.562718:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189571712, offset 0 00000400:00000200:1.0:1788971916.562723:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971916.562726:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971916.562736:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971916.562739:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971916.562741:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971916.562743:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971916.562746:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971916.562749:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [457:457:255:4294967295] 192.168.203.150@tcp LPNI seq info [457:457:7:4294967295] 00000400:00000200:1.0:1788971916.562754:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971916.562763:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971916.562768:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971916.562772:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f000. 00000800:00000200:1.0:1788971916.562776:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971916.562780:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971916.562783:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971916.562790:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971916.562796:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971916.562802:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f900. 02000000:00000010:1.0:1788971916.562808:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5c00. 00000400:00000200:3.0:1788971916.562809:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971916.562837:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971916.562843:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:3.0:1788971916.562847:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000400:00000200:1.0:1788971916.562856:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:3.0:1788971916.562859:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88e00 x1875872189571456/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971916.562861:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971916.562868:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788971916.562874:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571584, portal 10 00000400:00000010:1.0:1788971916.562880:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8. 00000100:00000200:1.0:1788971916.562883:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189571584, offset 0 00000400:00000200:1.0:1788971916.562888:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971916.562892:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971916.562896:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971916.562898:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971916.562901:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971916.562904:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971916.562906:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971916.562909:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [458:458:255:4294967295] 192.168.203.150@tcp LPNI seq info [458:458:7:4294967295] 00000400:00000200:1.0:1788971916.562913:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971916.562919:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971916.562929:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971916.562933:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f900. 00000800:00000200:1.0:1788971916.562938:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971916.562942:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000200:1.0:1788971916.562945:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000010:3.0:1788971916.562947:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f000. 00000800:00000200:1.0:1788971916.562949:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971916.562953:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788971916.562957:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.562962:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971916.562966:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525330 00000400:00000010:3.0:1788971916.562968:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525330. 00000100:00000200:3.0:1788971916.562974:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189571712/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971916.562990:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971916.562992:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f900. 00000400:00000200:3.0:1788971916.562995:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971916.562998:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971916.562999:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8 00000400:00000010:3.0:1788971916.563001:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8. 00000100:00000200:3.0:1788971916.563005:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189571584/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971916.565418:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971916.565430:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971916.565439:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971916.565445:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971916.565452:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971916.565456:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e180 00000400:00000200:2.0:1788971916.565463:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971916.565471:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x16bd [1] + 224 00000400:00000200:2.0:1788971916.565488:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971916.565492:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971916.565509:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88e00 x1875872189571456/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971916.567073:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971916.567086:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525110 00000400:00000010:1.0:1788971916.567088:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788971916.567102:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88e00 x1875872189571456/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971916.567114:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88e00 x1875872189571456/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971916.567140:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4800. 02000000:00000010:1.0:1788971916.567146:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be800. 00000100:00000010:1.0:1788971916.567150:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88e00. 00000400:00000200:3.0:1788971916.568604:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568616:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971916.568625:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568630:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568641:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568662:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e280 00000400:00000200:3.0:1788971916.568672:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568680:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x16c5 [1] + 224 00000400:00000200:3.0:1788971916.568694:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971916.568698:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971916.568718:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189571712/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971916.568764:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568767:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971916.568771:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568774:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568777:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.568780:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e200 00000400:00000200:3.0:1788971916.568783:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971916.569098:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x16cd [1] + 224 00000400:00000200:3.0:1788971916.569129:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971916.569133:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971916.569141:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189571584/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971916.569856:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971916.569868:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220 00000400:00000010:1.0:1788971916.569871:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788971916.569884:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189571584/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971916.569895:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189571584/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971916.569927:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000200:1.0:1788971916.569930:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:1.0:1788971916.569932:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971916.569937:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189571712/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971916.569944:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189571712/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971932 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971916.569977:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5c00. 02000000:00000010:1.0:1788971916.569983:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be200. 00000100:00000010:1.0:1788971916.569988:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 02000000:00000010:1.0:1788971916.569992:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5a00. 02000000:00000010:1.0:1788971916.569994:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09bef00. 00000100:00000010:1.0:1788971916.569996:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000100:00000010:2.0:1788971917.585969:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467a80. 00000100:00000010:3.0:1788971917.586148:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74677e0. 00000100:00000010:2.0:1788971921.682372:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f800. 02000000:00000010:2.0:1788971921.682409:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f400. 00000100:00000010:2.0:1788971921.682488:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788971921.682495:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f500. 00000100:00000010:2.0:1788971921.682503:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ce00. 02000000:00000010:2.0:1788971921.682507:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f000. 02000000:00000010:3.0:1788971921.682569:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2800. 00000400:00000200:3.0:1788971921.682584:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971921.682590:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788971921.682599:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971921.682609:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571840, portal 25 00000400:00000010:3.0:1788971921.682617:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788971921.682621:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189571840, offset 0 00000400:00000200:3.0:1788971921.682633:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971921.682644:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971921.682655:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971921.682669:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971921.682675:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971921.682680:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971921.682684:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971921.682688:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [459:459:256:4294967295] 192.168.203.150@tcp LPNI seq info [459:459:8:4294967295] 00000400:00000200:3.0:1788971921.682693:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971921.682704:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971921.682712:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971921.682718:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788971921.682726:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971921.682737:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971921.682743:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971921.682767:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971921.682773:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09300. 00000100:00000010:3.0:1788971921.682786:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09000. 02000000:00000010:3.0:1788971921.682801:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2e00. 00000400:00000200:3.0:1788971921.682805:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971921.682808:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788971921.682812:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971921.682854:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572096, portal 10 00000400:00000010:3.0:1788971921.682861:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788971921.682864:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189572096, offset 0 00000400:00000200:3.0:1788971921.682869:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971921.682873:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971921.682877:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971921.682881:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971921.682884:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971921.682886:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971921.682888:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971921.682891:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [460:460:255:4294967295] 192.168.203.150@tcp LPNI seq info [460:460:7:4294967295] 00000400:00000200:3.0:1788971921.682895:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971921.682908:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971921.682912:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971921.682916:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788971921.682921:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971921.682926:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971921.682929:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971921.682935:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971921.682956:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000010:3.0:1788971921.682959:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5a9a000. 00000800:00000010:2.0:1788971921.682963:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788971921.682972:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971921.682978:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971921.682986:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770 00000400:00000010:2.0:1788971921.682988:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770. 00000100:00000200:2.0:1788971921.682998:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f800 x1875872189571840/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971921.683027:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971921.683030:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788971921.683032:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971921.683035:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971921.683038:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28 00000400:00000010:2.0:1788971921.683040:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28. 00000100:00000200:2.0:1788971921.683044:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ce00 x1875872189572096/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971921.683076:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971921.683082:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68. 00000400:00000010:3.0:1788971921.683087:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873d48. 00000100:00000200:3.0:1788971921.683091:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189571968, portal 10 00000400:00000010:3.0:1788971921.683097:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873660. 00000100:00000200:3.0:1788971921.683100:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189571968, offset 0 00000400:00000200:3.0:1788971921.683104:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971921.683109:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971921.683112:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971921.683309:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971921.683312:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971921.683315:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971921.683318:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971921.683322:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [461:461:256:4294967295] 192.168.203.150@tcp LPNI seq info [461:461:8:4294967295] 00000400:00000200:3.0:1788971921.683326:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971921.683333:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971921.683338:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971921.683343:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00. 00000800:00000200:3.0:1788971921.683347:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971921.683352:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971921.683355:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971921.683376:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971921.683740:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971921.683744:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00. 00000400:00000200:2.0:1788971921.683749:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971921.683754:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971921.683758:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873660 00000400:00000010:2.0:1788971921.683760:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873660. 00000100:00000200:2.0:1788971921.683767:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189571968/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971921.690959:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.690970:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971921.690980:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691001:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691007:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691011:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e300 00000400:00000200:2.0:1788971921.691016:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691021:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x16d5 [1] + 224 00000400:00000200:2.0:1788971921.691033:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971921.691037:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971921.691051:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f800 x1875872189571840/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971921.691223:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691227:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971921.691232:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691234:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691237:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691240:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e400 00000400:00000200:2.0:1788971921.691244:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691248:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x16e5 [1] + 224 00000400:00000200:2.0:1788971921.691255:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971921.691259:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971921.691266:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ce00 x1875872189572096/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971921.691280:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691284:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971921.691288:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691290:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691293:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691295:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e380 00000400:00000200:2.0:1788971921.691298:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971921.691302:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x16f5 [1] + 224 00000400:00000200:2.0:1788971921.691312:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971921.691321:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971921.691327:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189571968/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971921.693933:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788971921.693945:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:3.0:1788971921.693948:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971921.693961:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f800 x1875872189571840/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971921.693972:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f800 x1875872189571840/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971921.693998:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:3.0:1788971921.694000:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788971921.694002:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971921.694006:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ce00 x1875872189572096/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971921.694016:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ce00 x1875872189572096/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971921.694034:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2800. 02000000:00000010:3.0:1788971921.694041:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f400. 00000100:00000010:3.0:1788971921.694046:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f800. 02000000:00000010:3.0:1788971921.694049:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2e00. 02000000:00000010:3.0:1788971921.694051:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f000. 00000100:00000010:3.0:1788971921.694053:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ce00. 00000400:00000010:3.0:1788971921.694059:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68. 00000400:00000200:3.0:1788971921.694061:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873d48 00000400:00000010:3.0:1788971921.694062:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873d48. 00000100:00000200:3.0:1788971921.694067:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189571968/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971921.694075:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189571968/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971937 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971921.694087:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5a9a000. 02000000:00000010:3.0:1788971921.694090:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f500. 00000100:00000010:3.0:1788971921.694092:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000100:00000010:2.0:1788971922.705942:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09300. 00000100:00000010:3.0:1788971922.707668:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09000. 00000100:00000010:3.0:1788971926.802170:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788971926.802201:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788971926.802236:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000. 02000000:00000010:3.0:1788971926.802240:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788971926.802246:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793480. 02000000:00000010:3.0:1788971926.802250:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 02000000:00000010:3.0:1788971926.802276:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5a9a000. 00000400:00000200:3.0:1788971926.802286:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971926.802292:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68. 02000000:00000010:2.0:1788971926.802292:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b000. 00000400:00000200:2.0:1788971926.802300:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971926.802301:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873d48. 00000400:00000010:2.0:1788971926.802306:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126aa8. 00000100:00000200:3.0:1788971926.802311:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572352, portal 10 00000400:00000010:2.0:1788971926.802313:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971926.802317:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572224, portal 25 00000400:00000010:2.0:1788971926.802322:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971926.802325:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189572224, offset 0 00000400:00000200:2.0:1788971926.802334:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971926.802341:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.802349:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971926.802353:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971926.802357:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971926.802361:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971926.802363:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971926.802377:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [462:462:256:4294967295] 192.168.203.150@tcp LPNI seq info [462:462:8:4294967295] 00000400:00000200:2.0:1788971926.802382:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.802391:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971926.802397:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971926.802401:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f900. 00000800:00000200:2.0:1788971926.802407:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971926.802422:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971926.802426:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971926.802438:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971926.802443:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e900. 00000100:00000010:2.0:1788971926.802461:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed80. 02000000:00000010:2.0:1788971926.802470:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788971926.802486:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971926.802488:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d68. 00000400:00000010:2.0:1788971926.802491:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788971926.802494:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572480, portal 10 00000400:00000010:2.0:1788971926.802499:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788971926.802501:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189572480, offset 0 00000400:00000200:2.0:1788971926.802505:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971926.802509:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.802523:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971926.802526:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971926.802528:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971926.802530:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971926.802533:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971926.802536:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [463:463:255:4294967295] 192.168.203.150@tcp LPNI seq info [463:463:7:4294967295] 00000400:00000200:2.0:1788971926.802539:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.802544:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971926.802548:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971926.802552:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00. 00000800:00000200:2.0:1788971926.802555:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971926.802559:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971926.802562:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971926.802567:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971926.802617:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971926.802621:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f900. 00000400:00000200:2.0:1788971926.802626:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971926.802631:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971926.802636:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:2.0:1788971926.802650:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971926.802660:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189572224/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971926.802737:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971926.802746:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00. 00000400:00000200:2.0:1788971926.802749:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971926.802752:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971926.802755:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788971926.802757:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788971926.802762:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793480 x1875872189572480/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971926.802779:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971926.802784:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189572352, offset 0 00000400:00000200:3.0:1788971926.802792:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971926.802797:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971926.802801:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971926.802804:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971926.802807:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971926.802810:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971926.802812:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971926.802835:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [464:464:256:4294967295] 192.168.203.150@tcp LPNI seq info [464:464:8:4294967295] 00000400:00000200:3.0:1788971926.802840:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971926.802846:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971926.802850:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971926.802854:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300. 00000800:00000200:3.0:1788971926.802858:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971926.802863:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971926.802866:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971926.803258:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826792:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826802:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971926.826825:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826831:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826837:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826841:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e580 00000400:00000200:2.0:1788971926.826847:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826853:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x171d [1] + 224 00000400:00000200:2.0:1788971926.826865:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971926.826869:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971926.826884:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793480 x1875872189572480/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971926.826920:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826924:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971926.826928:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826930:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826933:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826936:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e480 00000400:00000200:2.0:1788971926.826939:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.826943:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x170d [1] + 224 00000400:00000200:2.0:1788971926.826954:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971926.826958:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971926.826963:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189572224/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971926.827016:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126aa8. 00000400:00000200:2.0:1788971926.827020:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788971926.827023:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971926.827029:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189572224/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971926.827038:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189572224/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971926.827058:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d68. 00000400:00000200:2.0:1788971926.827061:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788971926.827063:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788971926.827067:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793480 x1875872189572480/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971926.827075:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793480 x1875872189572480/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971926.827093:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b000. 02000000:00000010:2.0:1788971926.827100:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00. 00000100:00000010:2.0:1788971926.827104:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 02000000:00000010:2.0:1788971926.827108:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:2.0:1788971926.827113:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788971926.827115:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793480. 00000800:00000200:3.0:1788971926.827361:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971926.827372:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300. 00000400:00000200:3.0:1788971926.827376:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971926.827380:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971926.827384:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788971926.827386:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971926.827394:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790000 x1875872189572352/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971926.830641:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.830651:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971926.830660:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.830665:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.830670:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.830674:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e500 00000400:00000200:2.0:1788971926.830679:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971926.830685:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1705 [1] + 224 00000400:00000200:2.0:1788971926.830698:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971926.830972:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971926.830991:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790000 x1875872189572352/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971926.831242:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68. 00000400:00000200:3.0:1788971926.831247:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873d48 00000400:00000010:3.0:1788971926.831250:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873d48. 00000100:00000200:3.0:1788971926.831257:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790000 x1875872189572352/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971926.831267:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790000 x1875872189572352/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971942 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971926.831298:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5a9a000. 02000000:00000010:3.0:1788971926.831305:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788971926.831309:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000. 00000100:00000010:2.0:1788971927.825982:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e900. 00000100:00000010:3.0:1788971927.826657:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed80. 00000100:00000010:2.0:1788971931.922240:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100. 02000000:00000010:2.0:1788971931.922263:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060ff00. 00000100:00000010:2.0:1788971931.922297:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788971931.922302:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f900. 00000100:00000010:2.0:1788971931.922308:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e300. 02000000:00000010:2.0:1788971931.922322:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f500. 02000000:00000010:2.0:1788971931.922352:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788971931.922361:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971931.922365:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d68. 00000400:00000010:2.0:1788971931.922370:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788971931.922376:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572736, portal 10 00000400:00000010:2.0:1788971931.922382:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971931.922384:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189572736, offset 0 00000400:00000200:2.0:1788971931.922392:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971931.922400:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.922407:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971931.922412:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971931.922415:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971931.922418:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971931.922421:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971931.922424:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [465:465:256:4294967295] 192.168.203.150@tcp LPNI seq info [465:465:8:4294967295] 00000400:00000200:2.0:1788971931.922428:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.922436:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971931.922441:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971931.922446:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f000. 00000800:00000200:2.0:1788971931.922452:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971931.922470:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971931.922475:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971931.922504:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971931.922510:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e900. 00000100:00000010:2.0:1788971931.922520:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e720. 02000000:00000010:2.0:1788971931.922547:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b000. 00000400:00000200:2.0:1788971931.922911:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971931.922915:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126aa8. 00000400:00000010:2.0:1788971931.922918:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788971931.922922:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572608, portal 25 00000400:00000010:2.0:1788971931.922926:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971931.922928:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189572608, offset 0 00000400:00000200:2.0:1788971931.922933:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971931.922937:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.922941:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971931.922944:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971931.922946:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971931.922948:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971931.922950:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971931.922953:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [466:466:255:4294967295] 192.168.203.150@tcp LPNI seq info [466:466:7:4294967295] 00000400:00000200:2.0:1788971931.922957:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.922962:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971931.922967:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971931.922970:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f400. 00000800:00000200:2.0:1788971931.922974:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971931.922978:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971931.922981:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971931.922985:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971931.922998:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b800. 00000400:00000200:2.0:1788971931.923001:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971931.923010:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126210. 00000400:00000010:2.0:1788971931.923014:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788971931.923017:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572864, portal 10 00000400:00000010:2.0:1788971931.923021:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8. 00000100:00000200:2.0:1788971931.923023:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189572864, offset 0 00000400:00000200:2.0:1788971931.923028:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971931.923032:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.923036:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971931.923038:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971931.923040:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971931.923042:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971931.923044:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971931.923047:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [467:467:254:4294967295] 192.168.203.150@tcp LPNI seq info [467:467:6:4294967295] 00000400:00000200:2.0:1788971931.923051:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.923056:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971931.923061:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971931.923064:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f100. 00000800:00000200:2.0:1788971931.923068:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971931.923072:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971931.923074:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971931.923079:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971931.923120:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971931.923123:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f000. 00000400:00000200:2.0:1788971931.923128:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971931.923132:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971931.923137:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788971931.923139:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971931.923159:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189572736/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971931.923180:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971931.923182:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f400. 00000400:00000200:2.0:1788971931.923190:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971931.923193:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971931.923196:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:2.0:1788971931.923198:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971931.923202:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f100 x1875872189572608/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971947 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971931.923268:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971931.923271:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f100. 00000400:00000200:2.0:1788971931.923274:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971931.923277:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971931.923279:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8 00000400:00000010:2.0:1788971931.923281:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8. 00000100:00000200:2.0:1788971931.923286:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e300 x1875872189572864/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971931.927462:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927538:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971931.927547:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927553:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927558:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927562:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e680 00000400:00000200:2.0:1788971931.927567:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927574:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1735 [1] + 224 00000400:00000200:2.0:1788971931.927586:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971931.927590:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971931.927603:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189572736/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971931.927686:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927690:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971931.927694:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927696:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927714:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927716:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e700 00000400:00000200:2.0:1788971931.927720:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.927723:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1755 [1] + 224 00000400:00000200:2.0:1788971931.927732:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971931.927736:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971931.927743:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e300 x1875872189572864/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971931.930160:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d68. 00000400:00000200:3.0:1788971931.930172:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:3.0:1788971931.930175:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:3.0:1788971931.930188:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189572736/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971931.930199:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189572736/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971931.930230:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:3.0:1788971931.930236:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f900. 00000100:00000010:3.0:1788971931.930240:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 00000400:00000010:3.0:1788971931.930248:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126210. 00000400:00000200:3.0:1788971931.930250:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:3.0:1788971931.930266:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:3.0:1788971931.930271:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e300 x1875872189572864/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971931.930292:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e300 x1875872189572864/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971931.930306:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b800. 02000000:00000010:3.0:1788971931.930311:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f500. 00000100:00000010:3.0:1788971931.930313:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e300. 00000400:00000200:2.0:1788971931.930530:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.930537:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971931.930545:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.930549:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.930554:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.930557:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e600 00000400:00000200:2.0:1788971931.930561:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971931.930567:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1745 [1] + 224 00000400:00000200:2.0:1788971931.930580:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971931.930585:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971931.930593:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f100 x1875872189572608/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971931.930677:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126aa8. 00000400:00000200:3.0:1788971931.930681:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:3.0:1788971931.930683:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:3.0:1788971931.930687:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f100 x1875872189572608/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971931.930766:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f100 x1875872189572608/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971947 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971931.930891:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b000. 02000000:00000010:3.0:1788971931.930896:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060ff00. 00000100:00000010:3.0:1788971931.930900:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100. 00000100:00000010:3.0:1788971932.945938:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e720. 00000100:00000010:2.0:1788971932.945938:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e900. 00000100:00000010:3.0:1788971937.042249:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000. 02000000:00000010:3.0:1788971937.042270:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788971937.042296:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100. 02000000:00000010:3.0:1788971937.042301:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300. 00000100:00000010:3.0:1788971937.042307:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788971937.042311:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 02000000:00000010:3.0:1788971937.042344:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5a9a000. 00000400:00000200:3.0:1788971937.042353:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971937.042357:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68. 00000400:00000010:3.0:1788971937.042365:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873d48. 00000100:00000200:3.0:1788971937.042372:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189572992, portal 25 00000400:00000010:3.0:1788971937.042391:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788971937.042394:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189572992, offset 0 00000400:00000200:3.0:1788971937.042404:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971937.042412:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971937.042420:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971937.042425:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971937.042429:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971937.042432:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971937.042435:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971937.042438:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [468:468:256:4294967295] 192.168.203.150@tcp LPNI seq info [468:468:8:4294967295] 00000400:00000200:3.0:1788971937.042442:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971937.042451:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971937.042457:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971937.042462:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67200. 00000800:00000200:3.0:1788971937.042468:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971937.042477:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971937.042481:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971937.042493:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971937.042498:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09f60. 00000100:00000010:3.0:1788971937.042508:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09a80. 02000000:00000010:3.0:1788971937.042521:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2e00. 00000400:00000200:3.0:1788971937.042524:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971937.042527:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788971937.042530:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971937.042533:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573248, portal 10 00000400:00000010:3.0:1788971937.042537:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873a18. 00000100:00000200:3.0:1788971937.042539:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189573248, offset 0 00000400:00000200:3.0:1788971937.042544:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971937.042547:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971937.042551:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971937.042554:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971937.042562:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971937.042564:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971937.042567:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971937.042570:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [469:469:255:4294967295] 192.168.203.150@tcp LPNI seq info [469:469:7:4294967295] 00000400:00000200:3.0:1788971937.042574:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971937.042580:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971937.042585:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971937.042588:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788971937.042592:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971937.042596:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971937.042598:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971937.042604:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788971937.042621:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1be2800. 00000400:00000200:3.0:1788971937.042625:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971937.042626:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788971937.042629:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788971937.042632:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573120, portal 10 00000400:00000010:3.0:1788971937.042636:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788971937.042638:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189573120, offset 0 00000400:00000200:3.0:1788971937.042643:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971937.042646:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971937.042649:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788971937.042653:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971937.042655:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788971937.042657:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971937.042659:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788971937.042661:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [470:470:254:4294967295] 192.168.203.150@tcp LPNI seq info [470:470:6:4294967295] 00000400:00000200:3.0:1788971937.042665:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971937.042669:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971937.042673:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788971937.042681:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788971937.042684:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788971937.042688:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788971937.042691:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971937.042695:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971937.044352:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971937.044360:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67200. 00000400:00000200:2.0:1788971937.044368:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971937.044375:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971937.044381:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:2.0:1788971937.044383:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:2.0:1788971937.044394:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790000 x1875872189572992/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971937.044420:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971937.044434:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:2.0:1788971937.044437:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971937.044441:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971937.044444:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873a18 00000400:00000010:2.0:1788971937.044446:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873a18. 00000100:00000200:2.0:1788971937.044451:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792300 x1875872189573248/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971937.044516:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971937.044519:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788971937.044522:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971937.044526:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971937.044529:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:2.0:1788971937.044530:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:2.0:1788971937.044536:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793100 x1875872189573120/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971937.051007:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051016:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971937.051025:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051030:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051035:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051039:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e880 00000400:00000200:2.0:1788971937.051044:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051049:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1775 [1] + 224 00000400:00000200:2.0:1788971937.051058:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971937.051062:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971937.051076:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792300 x1875872189573248/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971937.051121:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051124:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971937.051129:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051144:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051147:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051150:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e800 00000400:00000200:2.0:1788971937.051153:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.051157:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1785 [1] + 224 00000400:00000200:2.0:1788971937.051166:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971937.051169:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971937.051176:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793100 x1875872189573120/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971937.051322:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788971937.051332:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908 00000400:00000010:3.0:1788971937.051335:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788971937.051348:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793100 x1875872189573120/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971937.051370:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793100 x1875872189573120/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971937.055116:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:3.0:1788971937.055135:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:3.0:1788971937.055138:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971937.055155:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792300 x1875872189573248/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971937.055171:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792300 x1875872189573248/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971937.055175:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.055191:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971937.055203:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788971937.055208:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2800. 00000400:00000200:2.0:1788971937.055210:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971937.055220:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788971937.055222:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300. 00000400:00000200:2.0:1788971937.055229:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e780 00000100:00000010:3.0:1788971937.055231:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100. 02000000:00000010:3.0:1788971937.055239:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1be2e00. 00000400:00000200:2.0:1788971937.055240:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788971937.055246:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788971937.055251:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300. 00000400:00000200:2.0:1788971937.055253:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1765 [1] + 224 00000400:00000200:2.0:1788971937.055276:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971937.055281:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971937.055295:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790000 x1875872189572992/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971937.055356:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68. 00000400:00000200:2.0:1788971937.055361:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873d48 00000400:00000010:2.0:1788971937.055363:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873d48. 00000100:00000200:2.0:1788971937.055369:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790000 x1875872189572992/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971937.055377:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790000 x1875872189572992/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971953 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971937.055403:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5a9a000. 02000000:00000010:2.0:1788971937.055424:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788971937.055429:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000. 00000100:00000010:2.0:1788971938.065947:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09f60. 00000100:00000010:3.0:1788971938.066723:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09a80. 00000100:00000010:2.0:1788971942.164870:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788971942.164894:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f100. 00000100:00000010:2.0:1788971942.164942:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c380. 02000000:00000010:2.0:1788971942.164948:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f400. 00000100:00000010:2.0:1788971942.164954:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ea00. 02000000:00000010:2.0:1788971942.164957:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e060f000. 02000000:00000010:2.0:1788971942.164988:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094ba00. 00000400:00000200:2.0:1788971942.164997:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971942.165002:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126bb0. 00000400:00000010:2.0:1788971942.165008:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8. 00000100:00000200:2.0:1788971942.165015:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573504, portal 10 00000400:00000010:2.0:1788971942.165021:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971942.165024:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189573504, offset 0 00000400:00000200:2.0:1788971942.165032:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971942.165041:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.165049:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971942.165054:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971942.165058:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971942.165061:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971942.165063:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971942.165067:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [471:471:256:4294967295] 192.168.203.150@tcp LPNI seq info [471:471:8:4294967295] 00000400:00000200:2.0:1788971942.165081:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.165090:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971942.165096:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971942.165102:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060ff00. 00000800:00000200:2.0:1788971942.165108:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971942.165115:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971942.165120:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971942.165152:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788971942.165157:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e300. 00000100:00000010:2.0:1788971942.165167:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e840. 02000000:00000010:2.0:1788971942.165197:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788971942.165201:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971942.165204:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10. 00000400:00000010:2.0:1788971942.165207:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788971942.165210:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573376, portal 25 00000400:00000010:2.0:1788971942.165221:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550. 00000100:00000200:2.0:1788971942.165223:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189573376, offset 0 00000400:00000200:2.0:1788971942.165227:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971942.165231:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.165234:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971942.165236:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971942.165238:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971942.165240:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971942.165242:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971942.165245:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [472:472:255:4294967295] 192.168.203.150@tcp LPNI seq info [472:472:7:4294967295] 00000400:00000200:2.0:1788971942.165248:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.165253:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971942.165257:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971942.165260:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f500. 00000800:00000200:2.0:1788971942.165264:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971942.165268:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971942.165270:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971942.165274:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971942.165285:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094aa00. 00000400:00000200:2.0:1788971942.165288:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971942.165290:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126e18. 00000400:00000010:2.0:1788971942.165293:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b990. 00000100:00000200:2.0:1788971942.165296:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573632, portal 10 00000400:00000010:2.0:1788971942.165306:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788971942.165308:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189573632, offset 0 00000400:00000200:2.0:1788971942.165312:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971942.165316:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.165320:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971942.165323:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971942.165325:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971942.165327:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971942.165330:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971942.165332:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [473:473:254:4294967295] 192.168.203.150@tcp LPNI seq info [473:473:6:4294967295] 00000400:00000200:2.0:1788971942.165336:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.165341:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971942.165345:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971942.165348:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e060f900. 00000800:00000200:2.0:1788971942.165352:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971942.165356:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971942.165358:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e060f900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971942.165363:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971942.165406:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971942.165410:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060ff00. 00000400:00000200:2.0:1788971942.165416:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971942.165421:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971942.165427:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:2.0:1788971942.165429:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971942.165438:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c380 x1875872189573504/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971942.165459:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971942.165461:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f500. 00000400:00000200:2.0:1788971942.165463:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971942.165466:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971942.165469:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550 00000400:00000010:2.0:1788971942.165476:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550. 00000100:00000200:2.0:1788971942.165481:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189573376/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971942.165551:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971942.165554:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e060f900. 00000400:00000200:2.0:1788971942.165557:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971942.165560:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971942.165563:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:2.0:1788971942.165564:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788971942.165569:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ea00 x1875872189573632/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971942.171514:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171524:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971942.171533:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171538:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171543:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171547:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780e980 00000400:00000200:2.0:1788971942.171553:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171560:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1795 [1] + 224 00000400:00000200:2.0:1788971942.171572:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971942.171576:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971942.171591:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c380 x1875872189573504/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971942.171632:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171636:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971942.171640:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171642:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171645:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171647:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ea00 00000400:00000200:2.0:1788971942.171650:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171668:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x17b5 [1] + 224 00000400:00000200:2.0:1788971942.171675:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971942.171678:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971942.171684:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ea00 x1875872189573632/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971942.171698:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171701:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971942.171705:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171708:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171710:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171713:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780e900 00000400:00000200:2.0:1788971942.171716:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971942.171719:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x17a5 [1] + 224 00000400:00000200:2.0:1788971942.171728:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971942.171731:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000010:3.0:1788971942.171736:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126e18. 00000400:00000200:3.0:1788971942.171741:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b990 00000400:00000010:3.0:1788971942.171743:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b990. 00000100:00000200:3.0:1788971942.171750:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ea00 x1875872189573632/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971942.171750:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189573376/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971942.171765:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ea00 x1875872189573632/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971942.171790:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126bb0. 00000400:00000200:3.0:1788971942.171793:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8 00000400:00000010:3.0:1788971942.171795:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8. 00000100:00000200:3.0:1788971942.171799:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c380 x1875872189573504/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971942.171807:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c380 x1875872189573504/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971942.171852:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094aa00. 02000000:00000010:3.0:1788971942.171859:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f000. 00000100:00000010:3.0:1788971942.171863:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ea00. 02000000:00000010:3.0:1788971942.171867:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094ba00. 02000000:00000010:3.0:1788971942.171870:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f400. 00000100:00000010:3.0:1788971942.171872:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c380. 00000400:00000010:3.0:1788971942.171878:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10. 00000400:00000200:3.0:1788971942.171880:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:3.0:1788971942.171881:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:3.0:1788971942.171886:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189573376/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971942.171911:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189573376/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971958 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971942.171923:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:3.0:1788971942.171926:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e060f100. 00000100:00000010:3.0:1788971942.171929:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 00000100:00000010:3.0:1788971943.185932:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e840. 00000100:00000010:2.0:1788971943.186102:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e300. 00000100:00000010:0.0:1788971947.283521:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971947.283553:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09bee00. 00000100:00000010:0.0:1788971947.283608:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800. 02000000:00000010:0.0:1788971947.283635:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be700. 00000100:00000010:0.0:1788971947.283644:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788971947.283649:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be000. 02000000:00000010:0.0:1788971947.283694:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002a00. 00000400:00000200:0.0:1788971947.283705:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971947.283710:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4580. 00000400:00000010:0.0:1788971947.283718:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971947.283727:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573888, portal 10 00000400:00000010:0.0:1788971947.283734:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8198. 00000100:00000200:0.0:1788971947.283738:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189573888, offset 0 00000400:00000200:0.0:1788971947.283752:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971947.283762:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971947.283773:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971947.283780:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971947.283784:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971947.283789:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971947.283792:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971947.283797:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [474:474:256:4294967295] 192.168.203.150@tcp LPNI seq info [474:474:8:4294967295] 00000400:00000200:0.0:1788971947.283802:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971947.283835:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971947.283843:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971947.283859:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be100. 00000800:00000200:0.0:1788971947.283870:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971947.283882:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971947.283888:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971947.283915:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788971947.283921:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5360. 00000100:00000010:0.0:1788971947.283938:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5960. 02000000:00000010:0.0:1788971947.283973:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002800. 00000400:00000200:0.0:1788971947.283978:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971947.283981:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788971947.283990:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8d48. 00000100:00000200:0.0:1788971947.283993:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189573760, portal 25 00000400:00000010:0.0:1788971947.283997:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8a18. 00000100:00000200:0.0:1788971947.283999:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189573760, offset 0 00000400:00000200:0.0:1788971947.284005:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971947.284009:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971947.284012:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971947.284015:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971947.284018:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971947.284020:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971947.284022:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971947.284025:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [475:475:255:4294967295] 192.168.203.150@tcp LPNI seq info [475:475:7:4294967295] 00000400:00000200:0.0:1788971947.284029:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971947.284046:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971947.284051:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971947.284055:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09beb00. 00000800:00000200:0.0:1788971947.284059:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971947.284063:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971947.284066:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09beb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971947.284071:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971947.284150:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971947.284157:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be100. 00000400:00000200:2.0:1788971947.284166:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971947.284173:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971947.284180:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8198 00000400:00000010:2.0:1788971947.284182:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8198. 00000100:00000200:2.0:1788971947.284194:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b800 x1875872189573888/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971947.284285:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971947.284288:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09beb00. 00000400:00000200:2.0:1788971947.284291:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971947.284295:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971947.284305:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8a18 00000400:00000010:2.0:1788971947.284307:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8a18. 00000100:00000200:2.0:1788971947.284313:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88700 x1875872189573760/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971947.284334:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002400. 00000400:00000200:0.0:1788971947.284342:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971947.284345:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f78. 00000400:00000010:0.0:1788971947.284349:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8220. 00000100:00000200:0.0:1788971947.284353:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574016, portal 10 00000400:00000010:0.0:1788971947.284358:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8a18. 00000100:00000200:0.0:1788971947.284361:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189574016, offset 0 00000400:00000200:0.0:1788971947.284367:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971947.284372:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971947.284375:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971947.284378:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971947.284388:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971947.284390:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971947.284392:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971947.284395:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [476:476:256:4294967295] 192.168.203.150@tcp LPNI seq info [476:476:8:4294967295] 00000400:00000200:0.0:1788971947.284400:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971947.284406:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971947.284411:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971947.284414:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09beb00. 00000800:00000200:0.0:1788971947.284418:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971947.284423:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971947.284425:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09beb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971947.284443:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971947.286307:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971947.286319:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09beb00. 00000400:00000200:2.0:1788971947.286328:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971947.286338:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971947.286346:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8a18 00000400:00000010:2.0:1788971947.286350:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8a18. 00000100:00000200:2.0:1788971947.286361:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88000 x1875872189574016/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971947.289862:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289872:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971947.289882:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289887:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289892:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289896:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780eb00 00000400:00000200:2.0:1788971947.289902:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289907:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x17c5 [1] + 224 00000400:00000200:2.0:1788971947.289919:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971947.289923:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971947.289939:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b800 x1875872189573888/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971947.289969:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289973:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971947.289977:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289980:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289983:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.289986:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780eb80 00000400:00000200:2.0:1788971947.290004:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971947.290012:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4580. 00000400:00000200:0.0:1788971947.290026:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:0.0:1788971947.290029:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000400:00000200:2.0:1788971947.290054:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x17d5 [1] + 224 00000100:00000200:0.0:1788971947.290060:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b800 x1875872189573888/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971947.290067:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971947.290080:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:0.0:1788971947.290086:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b800 x1875872189573888/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971947.290094:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88000 x1875872189574016/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971947.290126:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.290131:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971947.290136:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.290138:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.290141:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.290144:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780ea80 00000400:00000200:2.0:1788971947.290150:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971947.290154:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x17cd [1] + 224 00000400:00000200:2.0:1788971947.290172:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971947.290176:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971947.290182:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88700 x1875872189573760/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971947.290556:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002a00. 02000000:00000010:0.0:1788971947.290567:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be700. 00000100:00000010:0.0:1788971947.290573:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800. 00000400:00000010:0.0:1788971947.290583:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788971947.290586:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8d48 00000400:00000010:0.0:1788971947.290588:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8d48. 00000100:00000200:0.0:1788971947.290603:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88700 x1875872189573760/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971947.290614:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88700 x1875872189573760/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971947.290629:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f78. 00000400:00000200:0.0:1788971947.290647:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8220 00000400:00000010:0.0:1788971947.290649:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8220. 00000100:00000200:0.0:1788971947.290654:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88000 x1875872189574016/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971947.290663:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88000 x1875872189574016/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971963 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971947.290678:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002800. 02000000:00000010:0.0:1788971947.290681:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09bee00. 00000100:00000010:0.0:1788971947.290683:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971947.290686:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002400. 02000000:00000010:0.0:1788971947.290689:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be000. 00000100:00000010:0.0:1788971947.290691:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88000. 00000100:00000010:2.0:1788971948.305957:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5360. 00000100:00000010:3.0:1788971948.306305:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5960. 00000100:00000010:1.0:1788971952.402273:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788971952.402309:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f800. 00000100:00000010:1.0:1788971952.402350:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700. 02000000:00000010:1.0:1788971952.402356:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fb00. 00000100:00000010:1.0:1788971952.402362:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788971952.402366:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00. 02000000:00000010:0.0:1788971952.402526:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002400. 00000400:00000200:0.0:1788971952.402542:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971952.402548:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f78. 00000400:00000010:0.0:1788971952.402557:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8220. 00000100:00000200:0.0:1788971952.402566:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574144, portal 25 00000400:00000010:0.0:1788971952.402574:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8d48. 00000100:00000200:0.0:1788971952.402578:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189574144, offset 0 00000400:00000200:0.0:1788971952.402590:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971952.402603:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971952.402615:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971952.402621:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971952.402627:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971952.402631:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971952.402634:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971952.402639:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [477:477:256:4294967295] 192.168.203.150@tcp LPNI seq info [477:477:8:4294967295] 00000400:00000200:0.0:1788971952.402643:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971952.402660:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971952.402666:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971952.402674:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be000. 00000800:00000200:0.0:1788971952.402702:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971952.402712:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971952.402719:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971952.402790:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788971952.402799:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5960. 00000100:00000010:0.0:1788971952.402849:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5360. 02000000:00000010:0.0:1788971952.402865:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002800. 00000400:00000200:0.0:1788971952.402869:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971952.402873:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788971952.402877:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971952.402880:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574400, portal 10 00000400:00000010:0.0:1788971952.402886:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8198. 00000100:00000200:0.0:1788971952.402889:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189574400, offset 0 00000400:00000200:0.0:1788971952.402894:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971952.402898:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971952.402902:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971952.402905:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971952.402908:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971952.402910:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971952.402913:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971952.402916:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [478:478:255:4294967295] 192.168.203.150@tcp LPNI seq info [478:478:7:4294967295] 00000400:00000200:0.0:1788971952.402919:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971952.402925:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971952.402930:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971952.402934:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09bee00. 00000800:00000200:0.0:1788971952.402938:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971952.402943:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971952.402945:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09bee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971952.402952:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788971952.402975:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002a00. 00000400:00000200:0.0:1788971952.402986:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971952.402988:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4580. 00000400:00000010:0.0:1788971952.402992:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a86e8. 00000100:00000200:0.0:1788971952.403144:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574272, portal 10 00000400:00000010:0.0:1788971952.403153:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8440. 00000100:00000200:0.0:1788971952.403157:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189574272, offset 0 00000400:00000200:0.0:1788971952.403163:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971952.403171:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971952.403175:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971952.403178:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971952.403180:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971952.403183:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971952.403185:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971952.403187:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [479:479:254:4294967295] 192.168.203.150@tcp LPNI seq info [479:479:6:4294967295] 00000400:00000200:0.0:1788971952.403192:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971952.403198:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971952.403203:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971952.403207:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be700. 00000800:00000200:0.0:1788971952.403211:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971952.403216:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971952.403219:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971952.403224:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971952.406396:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971952.406411:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be000. 00000400:00000200:2.0:1788971952.406423:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971952.406435:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971952.406445:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8d48 00000400:00000010:2.0:1788971952.406447:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8d48. 00000100:00000200:2.0:1788971952.406461:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189574144/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971952.406493:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971952.406495:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09bee00. 00000400:00000200:2.0:1788971952.406509:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971952.406533:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971952.406537:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8198 00000400:00000010:2.0:1788971952.406539:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8198. 00000100:00000200:2.0:1788971952.406546:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189574400/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971952.407226:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971952.407232:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be700. 00000400:00000200:2.0:1788971952.407237:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971952.407242:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971952.407247:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8440 00000400:00000010:2.0:1788971952.407250:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8440. 00000100:00000200:2.0:1788971952.407256:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f0700 x1875872189574272/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971952.414773:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414786:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971952.414796:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414802:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414810:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414849:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ed00 00000400:00000200:2.0:1788971952.414856:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414864:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x17e5 [1] + 224 00000400:00000200:2.0:1788971952.414878:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971952.414883:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971952.414921:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189574400/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971952.414965:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414969:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971952.414973:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414976:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.414978:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415011:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ec80 00000400:00000200:2.0:1788971952.415015:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415020:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x17ed [1] + 224 00000400:00000200:2.0:1788971952.415028:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971952.415031:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971952.415048:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f0700 x1875872189574272/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971952.415066:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415069:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971952.415074:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415076:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415079:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415082:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780ec00 00000400:00000200:2.0:1788971952.415086:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971952.415090:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x17dd [1] + 224 00000400:00000200:2.0:1788971952.415100:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971952.415104:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971952.415111:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189574144/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971952.416132:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4580. 00000400:00000200:0.0:1788971952.416142:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a86e8 00000400:00000010:0.0:1788971952.416146:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a86e8. 00000100:00000200:0.0:1788971952.416158:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f0700 x1875872189574272/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971952.416170:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f0700 x1875872189574272/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971952.416209:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f78. 00000400:00000200:0.0:1788971952.416224:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8220 00000400:00000010:0.0:1788971952.416226:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8220. 00000100:00000200:0.0:1788971952.416233:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189574144/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971952.416242:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189574144/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971952.416257:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788971952.416260:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:0.0:1788971952.416261:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971952.416265:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189574400/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971952.416273:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189574400/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971968 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971952.416294:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002a00. 02000000:00000010:0.0:1788971952.416303:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fb00. 00000100:00000010:0.0:1788971952.416309:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700. 02000000:00000010:0.0:1788971952.416313:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002400. 02000000:00000010:0.0:1788971952.416315:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f800. 00000100:00000010:0.0:1788971952.416318:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 02000000:00000010:0.0:1788971952.416321:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002800. 02000000:00000010:0.0:1788971952.416324:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00. 00000100:00000010:0.0:1788971952.416326:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000100:00000010:3.0:1788971953.425943:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5360. 00000100:00000010:2.0:1788971953.426049:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5960. 00000100:00000010:1.0:1788971957.521993:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80. 02000000:00000010:1.0:1788971957.522015:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f600. 00000100:00000010:1.0:1788971957.522053:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3b80. 02000000:00000010:1.0:1788971957.522058:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0ff00. 00000100:00000010:1.0:1788971957.522063:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800. 02000000:00000010:1.0:1788971957.522068:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00. 02000000:00000010:0.0:1788971957.522124:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002800. 00000400:00000200:0.0:1788971957.522135:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971957.522139:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788971957.522145:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971957.522151:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574528, portal 25 00000400:00000010:0.0:1788971957.522156:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8220. 00000100:00000200:0.0:1788971957.522159:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189574528, offset 0 00000400:00000200:0.0:1788971957.522168:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971957.522176:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971957.522195:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971957.522201:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971957.522205:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971957.522209:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971957.522212:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971957.522215:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [480:480:256:4294967295] 192.168.203.150@tcp LPNI seq info [480:480:8:4294967295] 00000400:00000200:0.0:1788971957.522219:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971957.522228:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971957.522233:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971957.522239:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be100. 00000800:00000200:0.0:1788971957.522245:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971957.522253:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971957.522258:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971957.522280:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788971957.522285:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c56c0. 00000100:00000010:0.0:1788971957.522295:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5720. 02000000:00000010:0.0:1788971957.522308:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002400. 00000400:00000200:0.0:1788971957.522312:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971957.522315:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f78. 00000400:00000010:0.0:1788971957.522319:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a86e8. 00000100:00000200:0.0:1788971957.522322:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574784, portal 10 00000400:00000010:0.0:1788971957.522327:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8770. 00000100:00000200:0.0:1788971957.522330:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189574784, offset 0 00000400:00000200:0.0:1788971957.522335:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971957.522338:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971957.522395:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971957.522400:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be100. 00000400:00000200:2.0:1788971957.522405:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971957.522412:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:0.0:1788971957.522418:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971957.522419:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8220 00000400:00000010:2.0:1788971957.522424:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8220. 00000400:00000200:0.0:1788971957.522425:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971957.522430:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971957.522436:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971957.522439:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971957.522441:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [481:481:255:4294967295] 192.168.203.150@tcp LPNI seq info [481:481:7:4294967295] 00000100:00000200:2.0:1788971957.522446:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1f80 x1875872189574528/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788971957.522460:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971957.522471:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971957.522476:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971957.522480:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09bef00. 00000800:00000200:0.0:1788971957.522484:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971957.522489:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971957.522492:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09bef00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971957.522524:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788971957.522543:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002a00. 00000400:00000200:0.0:1788971957.522547:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971957.522549:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4580. 00000400:00000010:0.0:1788971957.522553:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788971957.522556:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574656, portal 10 00000400:00000010:0.0:1788971957.522561:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8ee0. 00000100:00000200:0.0:1788971957.522562:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189574656, offset 0 00000400:00000200:0.0:1788971957.522566:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971957.522570:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971957.522573:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971957.522575:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971957.522577:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971957.522579:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971957.522581:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971957.522584:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [482:482:255:4294967295] 192.168.203.150@tcp LPNI seq info [482:482:7:4294967295] 00000400:00000200:0.0:1788971957.522587:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971957.522592:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971957.522596:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971957.522599:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be200. 00000800:00000200:0.0:1788971957.522602:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971957.522607:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971957.522609:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971957.522621:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971957.522857:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971957.522861:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09bef00. 00000400:00000200:3.0:1788971957.522864:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971957.522868:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971957.522871:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8770 00000400:00000010:3.0:1788971957.522873:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8770. 00000100:00000200:3.0:1788971957.522880:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3800 x1875872189574784/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971957.522925:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971957.522928:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be200. 00000400:00000200:3.0:1788971957.522931:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971957.522935:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971957.522937:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8ee0 00000400:00000010:3.0:1788971957.522939:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8ee0. 00000100:00000200:3.0:1788971957.522944:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3b80 x1875872189574656/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971957.525237:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971957.525247:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971957.525278:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971957.525282:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971957.525287:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971957.525290:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780ed80 00000400:00000200:2.0:1788971957.525295:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971957.525300:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x17f5 [1] + 224 00000400:00000200:2.0:1788971957.525314:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971957.525318:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971957.525331:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1f80 x1875872189574528/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971957.526477:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526487:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971957.526495:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526500:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526505:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526508:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ee80 00000400:00000200:3.0:1788971957.526768:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526778:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x17fd [1] + 224 00000400:00000200:3.0:1788971957.526793:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971957.526798:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971957.526812:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3800 x1875872189574784/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971957.526883:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526886:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971957.526890:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526908:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526910:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526913:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ee00 00000400:00000200:3.0:1788971957.526915:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971957.526919:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1805 [1] + 224 00000400:00000200:3.0:1788971957.526927:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971957.526930:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971957.526936:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3b80 x1875872189574656/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971957.528181:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788971957.528192:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:0.0:1788971957.528196:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971957.528209:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1f80 x1875872189574528/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971957.528220:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1f80 x1875872189574528/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971957.528246:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f78. 00000400:00000200:0.0:1788971957.528249:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a86e8 00000400:00000010:0.0:1788971957.528251:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a86e8. 00000100:00000200:0.0:1788971957.528265:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3800 x1875872189574784/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971957.528273:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3800 x1875872189574784/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971957.528294:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002800. 02000000:00000010:0.0:1788971957.528301:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f600. 00000100:00000010:0.0:1788971957.528306:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80. 02000000:00000010:0.0:1788971957.528311:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002400. 02000000:00000010:0.0:1788971957.528313:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00. 00000100:00000010:0.0:1788971957.528315:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800. 00000400:00000010:0.0:1788971957.528321:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4580. 00000400:00000200:0.0:1788971957.528323:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38 00000400:00000010:0.0:1788971957.528325:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788971957.528330:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3b80 x1875872189574656/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971957.528338:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3b80 x1875872189574656/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971973 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971957.528351:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002a00. 02000000:00000010:0.0:1788971957.528354:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0ff00. 00000100:00000010:0.0:1788971957.528371:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3b80. 00000100:00000010:2.0:1788971958.545948:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c56c0. 00000100:00000010:3.0:1788971958.545964:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5720. 00000100:00000010:0.0:1788971962.643126:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788971962.643154:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be800. 00000100:00000010:0.0:1788971962.643204:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971962.643211:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be300. 00000100:00000010:0.0:1788971962.643223:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800. 02000000:00000010:0.0:1788971962.643228:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be400. 02000000:00000010:0.0:1788971962.643270:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e0b1ae00. 00000400:00000200:0.0:1788971962.643283:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971962.643288:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4580. 00000400:00000010:0.0:1788971962.643297:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788971962.643306:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189574912, portal 25 00000400:00000010:0.0:1788971962.643311:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a86e8. 00000100:00000200:0.0:1788971962.643317:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189574912, offset 0 00000400:00000200:0.0:1788971962.643330:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971962.643342:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971962.643353:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971962.643361:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971962.643367:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971962.643373:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971962.643378:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971962.643384:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [483:483:256:4294967295] 192.168.203.150@tcp LPNI seq info [483:483:8:4294967295] 00000400:00000200:0.0:1788971962.643389:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971962.643407:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971962.643416:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971962.643421:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09bed00. 00000800:00000200:0.0:1788971962.643431:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971962.643441:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971962.643448:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09bed00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971962.644356:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788971962.644365:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5840. 00000100:00000010:0.0:1788971962.644393:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c52a0. 02000000:00000010:0.0:1788971962.644413:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e0b1a800. 00000400:00000200:0.0:1788971962.644419:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971962.644423:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4f78. 00000400:00000010:0.0:1788971962.644429:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971962.644435:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189575168, portal 10 00000400:00000010:0.0:1788971962.644443:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8908. 00000100:00000200:0.0:1788971962.644446:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189575168, offset 0 00000400:00000200:0.0:1788971962.644454:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971962.644460:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971962.644465:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971962.644469:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971962.644472:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971962.644476:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971962.644479:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971962.644482:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [484:484:255:4294967295] 192.168.203.150@tcp LPNI seq info [484:484:7:4294967295] 00000400:00000200:0.0:1788971962.644486:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971962.644494:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971962.644499:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971962.644504:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be200. 00000800:00000200:0.0:1788971962.644510:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971962.644515:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971962.644531:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971962.645347:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788971962.645384:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e0b1aa00. 00000400:00000200:0.0:1788971962.645391:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971962.645394:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788971962.645402:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788971962.645408:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189575040, portal 10 00000400:00000010:0.0:1788971962.645415:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8660. 00000100:00000200:0.0:1788971962.645418:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189575040, offset 0 00000400:00000200:0.0:1788971962.645436:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971962.645442:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971962.645447:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971962.645451:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971962.645455:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971962.645458:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971962.645461:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971962.645465:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [485:485:254:4294967295] 192.168.203.150@tcp LPNI seq info [485:485:6:4294967295] 00000400:00000200:0.0:1788971962.645468:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971962.645476:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971962.645482:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971962.645487:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09bef00. 00000800:00000200:0.0:1788971962.645492:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971962.645499:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971962.645503:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09bef00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971962.645509:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971962.647671:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971962.647685:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09bed00. 00000400:00000200:3.0:1788971962.647694:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971962.647704:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971962.647713:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a86e8 00000400:00000010:3.0:1788971962.647716:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a86e8. 00000100:00000200:3.0:1788971962.647730:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88000 x1875872189574912/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971962.647763:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971962.647765:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be200. 00000400:00000200:3.0:1788971962.647768:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971962.647772:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971962.647775:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8908 00000400:00000010:3.0:1788971962.647777:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8908. 00000100:00000200:3.0:1788971962.647783:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b800 x1875872189575168/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971962.649218:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971962.649228:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09bef00. 00000400:00000200:3.0:1788971962.649235:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971962.649243:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971962.649250:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8660 00000400:00000010:3.0:1788971962.649260:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8660. 00000100:00000200:3.0:1788971962.649271:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88700 x1875872189575040/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971962.649315:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649322:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971962.649331:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649336:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649343:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649347:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780ef00 00000400:00000200:3.0:1788971962.649353:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649360:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x180d [1] + 224 00000400:00000200:3.0:1788971962.649434:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971962.649440:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971962.649448:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88000 x1875872189574912/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971962.649542:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649547:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971962.649552:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649555:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649558:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649560:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ef80 00000400:00000200:3.0:1788971962.649564:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649569:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x181d [1] + 224 00000400:00000200:3.0:1788971962.649617:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971962.649621:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971962.649627:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88700 x1875872189575040/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971962.649641:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649655:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971962.649666:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649669:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649671:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649674:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f000 00000400:00000200:3.0:1788971962.649677:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971962.649681:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1815 [1] + 224 00000400:00000200:3.0:1788971962.649689:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971962.649692:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971962.649699:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b800 x1875872189575168/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971962.650116:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4580. 00000400:00000200:0.0:1788971962.650124:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38 00000400:00000010:0.0:1788971962.650127:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788971962.650134:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88000 x1875872189574912/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971962.650144:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88000 x1875872189574912/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971962.650175:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4f78. 00000400:00000200:0.0:1788971962.650178:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:0.0:1788971962.650180:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788971962.650186:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b800 x1875872189575168/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971962.650196:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b800 x1875872189575168/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971962.650234:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e0b1ae00. 02000000:00000010:0.0:1788971962.650242:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be800. 00000100:00000010:0.0:1788971962.650249:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788971962.650254:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e0b1a800. 02000000:00000010:0.0:1788971962.650258:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be400. 00000100:00000010:0.0:1788971962.650261:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800. 00000400:00000010:0.0:1788971962.650269:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788971962.650273:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788971962.650275:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788971962.650281:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88700 x1875872189575040/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971962.650289:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88700 x1875872189575040/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971978 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971962.650304:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e0b1aa00. 02000000:00000010:0.0:1788971962.650307:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be300. 00000100:00000010:0.0:1788971962.650311:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700. 00000001:02000400:1.0:1788971962.674992:0:112836:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_300: @@@@@@ FAIL: test_300 failed with 2 00000100:00000010:3.0:1788971963.666030:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c52a0. 00000100:00000010:2.0:1788971963.666451:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5840. Debug log: 4367 lines, 4367 kept, 0 dropped, 0 bad.