00000100:00000010:0.0F:1788972155.847621:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972155.847670:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788972155.847727:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972155.847733:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:0.0:1788972155.847823:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa000. 00000400:00000200:0.0:1788972155.847835:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972155.847840:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163268. 00000400:00000010:0.0:1788972155.847849:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972155.847858:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179081856, portal 10 00000400:00000010:0.0:1788972155.847868:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084330. 00000100:00000200:0.0:1788972155.847872:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179081856, offset 0 00000400:00000200:0.0:1788972155.847882:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972155.847893:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972155.847903:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972155.847909:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972155.847915:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972155.847919:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972155.847922:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972155.847933:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [425:425:256:4294967295] 192.168.203.124@tcp LPNI seq info [425:425:8:4294967295] 00000400:00000200:0.0:1788972155.847939:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972155.847950:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972155.847958:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972155.847965:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000. 00000800:00000200:0.0:1788972155.847973:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972155.847982:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972155.847989:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972155.848016:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972155.848023:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969960. 00000100:00000010:0.0:1788972155.848040:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969b40. 02000000:00000010:0.0:1788972155.848064:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aba00. 00000400:00000200:0.0:1788972155.848069:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972155.848072:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163dc0. 00000400:00000010:0.0:1788972155.848082:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084770. 00000100:00000200:0.0:1788972155.848085:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179081728, portal 25 00000400:00000010:0.0:1788972155.848090:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972155.848092:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179081728, offset 0 00000400:00000200:0.0:1788972155.848097:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972155.848102:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972155.848105:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972155.848108:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972155.848110:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972155.848112:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972155.848114:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972155.848117:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [426:426:255:4294967295] 192.168.203.124@tcp LPNI seq info [426:426:7:4294967295] 00000400:00000200:0.0:1788972155.848120:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972155.848126:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972155.848130:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972155.848134:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55400. 00000800:00000200:0.0:1788972155.848137:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972155.848142:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972155.848144:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972155.848148:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0F:1788972155.848593:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972155.848606:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000. 00000400:00000200:3.0:1788972155.848616:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972155.848622:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972155.848629:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084330 00000400:00000010:3.0:1788972155.848632:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084330. 00000100:00000200:3.0:1788972155.848645:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179081856/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972155.848735:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972155.848738:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55400. 00000400:00000200:3.0:1788972155.848741:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972155.848744:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972155.848747:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:3.0:1788972155.848749:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000100:00000200:3.0:1788972155.848753:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179081728/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972171 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0F:1788972155.852262:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852289:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972155.852335:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852342:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852349:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852353:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d280 00000400:00000200:2.0:1788972155.852360:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852367:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1675 [1] + 224 00000400:00000200:2.0:1788972155.852393:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972155.852399:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972155.852416:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179081856/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972171 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972155.852449:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852453:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972155.852458:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852460:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852463:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852466:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d200 00000400:00000200:2.0:1788972155.852470:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972155.852474:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x167d [1] + 224 00000400:00000200:2.0:1788972155.852483:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972155.852487:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972155.852494:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179081728/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972171 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972155.852712:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163dc0. 00000400:00000200:0.0:1788972155.852718:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084770 00000400:00000010:0.0:1788972155.852722:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084770. 00000100:00000200:0.0:1788972155.852731:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179081728/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972171 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972155.852741:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179081728/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972171 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972155.852772:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163268. 00000400:00000200:0.0:1788972155.852775:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:0.0:1788972155.852777:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972155.852781:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179081856/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972171 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972155.852799:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179081856/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972171 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972155.852825:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aba00. 02000000:00000010:0.0:1788972155.852833:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788972155.852839:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972155.852843:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa000. 02000000:00000010:0.0:1788972155.852846:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788972155.852847:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 00000100:00000010:3.0:1788972156.551444:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969b40. 00000100:00000010:2.0:1788972156.551549:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969960. 00000100:00000010:3.0:1788972160.967492:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5f80. 02000000:00000010:3.0:1788972160.967519:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00. 00000100:00000010:3.0:1788972160.967555:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5500. 02000000:00000010:3.0:1788972160.967560:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 02000000:00000010:3.0:1788972160.967594:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96e00. 00000400:00000200:3.0:1788972160.967603:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972160.967608:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e785d8. 00000400:00000010:3.0:1788972160.967615:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972160.967622:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082112, portal 10 00000400:00000010:3.0:1788972160.967629:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972160.967632:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179082112, offset 0 00000400:00000200:3.0:1788972160.967642:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972160.967650:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.967657:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972160.967663:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972160.967668:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972160.967687:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972160.967690:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972160.967694:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [427:427:256:4294967295] 192.168.203.124@tcp LPNI seq info [427:427:8:4294967295] 00000400:00000200:3.0:1788972160.967699:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.967710:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972160.967717:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972160.967722:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:3.0:1788972160.967729:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972160.967737:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972160.967741:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972160.967755:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972160.967759:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91420. 00000100:00000010:3.0:1788972160.967768:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91540. 02000000:00000010:3.0:1788972160.967786:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97000. 00000400:00000200:3.0:1788972160.967790:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972160.967792:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78cb8. 00000400:00000010:3.0:1788972160.967796:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788972160.967798:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179081984, portal 25 00000400:00000010:3.0:1788972160.967801:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788972160.967803:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179081984, offset 0 00000400:00000200:3.0:1788972160.967807:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972160.967810:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.967813:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972160.967816:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972160.967818:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972160.967820:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972160.967822:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972160.967825:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [428:428:255:4294967295] 192.168.203.124@tcp LPNI seq info [428:428:7:4294967295] 00000400:00000200:3.0:1788972160.967838:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.967844:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972160.967856:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972160.967859:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500. 00000800:00000200:3.0:1788972160.967863:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972160.967867:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972160.967870:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972160.967874:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972160.967911:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972160.967915:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:3.0:1788972160.967919:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972160.967923:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972160.969355:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788972160.969366:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972160.969378:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5500 x1875872179082112/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972160.970220:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972160.970225:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500. 00000400:00000200:3.0:1788972160.970941:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972160.970949:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972160.970955:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788972160.970957:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788972160.970966:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5f80 x1875872179081984/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972176 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972160.972431:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.972438:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972160.972445:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.972449:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.972453:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.972457:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d380 00000400:00000200:3.0:1788972160.972473:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.972479:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1685 [1] + 224 00000400:00000200:3.0:1788972160.972500:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972160.972505:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972160.972515:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5500 x1875872179082112/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972176 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972160.972558:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e785d8. 00000400:00000200:3.0:1788972160.972562:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972160.972564:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972160.972569:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5500 x1875872179082112/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972176 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972160.972578:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5500 x1875872179082112/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972176 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972160.972606:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96e00. 02000000:00000010:3.0:1788972160.972610:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:3.0:1788972160.972615:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5500. 00000400:00000200:3.0:1788972160.974399:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.974410:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972160.974416:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.974420:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.974423:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.974427:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d300 00000400:00000200:3.0:1788972160.974431:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972160.974436:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1695 [1] + 224 00000400:00000200:3.0:1788972160.974445:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972160.974449:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972160.974459:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5f80 x1875872179081984/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972176 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972160.980038:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78cb8. 00000400:00000200:2.0:1788972160.980049:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:2.0:1788972160.980052:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:2.0:1788972160.980069:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5f80 x1875872179081984/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972176 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972160.980091:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5f80 x1875872179081984/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972176 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972160.980131:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97000. 02000000:00000010:2.0:1788972160.980138:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00. 00000100:00000010:2.0:1788972160.980143:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5f80. 00000001:02000400:0.0:1788972161.181992:0:114551:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 304: Check locked primary peer nid consolidation ========================================================== 12:42:40 (1788972160) 00000020:00000010:1.0F:1788972161.426404:0:114575:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972161.426512:0:114575:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:3.0:1788972161.468817:0:114580:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972161.468904:0:114580:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000100:00000010:3.0:1788972161.671741:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91540. 00000100:00000010:2.0:1788972161.671741:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91420. 00000020:00000010:2.0:1788972162.510634:0:114584:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972162.510722:0:114584:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:3.0:1788972163.540694:0:114588:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972163.540778:0:114588:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:0.0:1788972164.568558:0:114592:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972164.570346:0:114592:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:2.0:1788972165.631508:0:114596:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972165.631601:0:114596:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:1.0:1788972166.087517:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb5180. 02000000:00000010:1.0:1788972166.087540:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972166.087582:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6300. 02000000:00000010:1.0:1788972166.087589:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 02000000:00000010:0.0:1788972166.087685:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a000. 00000400:00000200:0.0:1788972166.087695:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972166.087699:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163268. 00000400:00000010:0.0:1788972166.087705:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972166.087712:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082240, portal 25 00000400:00000010:0.0:1788972166.087717:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084770. 00000100:00000200:0.0:1788972166.087720:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179082240, offset 0 00000400:00000200:0.0:1788972166.087728:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972166.087736:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972166.087744:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972166.087749:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972166.087753:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972166.087762:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972166.087765:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972166.087769:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [429:429:256:4294967295] 192.168.203.124@tcp LPNI seq info [429:429:8:4294967295] 00000400:00000200:0.0:1788972166.087773:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972166.087783:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972166.087807:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972166.087812:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800. 00000800:00000200:0.0:1788972166.087819:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972166.087826:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972166.087831:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972166.087854:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972166.087859:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969540. 00000100:00000010:0.0:1788972166.087870:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969ae0. 02000000:00000010:0.0:1788972166.087894:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166be00. 00000400:00000200:0.0:1788972166.087899:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972166.087901:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163dc0. 00000400:00000010:0.0:1788972166.087905:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972166.087908:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082368, portal 10 00000400:00000010:0.0:1788972166.087913:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084330. 00000100:00000200:0.0:1788972166.087916:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179082368, offset 0 00000400:00000200:0.0:1788972166.087920:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972166.087924:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972166.087928:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972166.087930:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972166.087932:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972166.087935:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972166.087937:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972166.087940:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [430:430:255:4294967295] 192.168.203.124@tcp LPNI seq info [430:430:7:4294967295] 00000400:00000200:0.0:1788972166.087944:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972166.087950:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972166.087954:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972166.087957:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55300. 00000800:00000200:0.0:1788972166.087961:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972166.087965:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972166.087968:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972166.087980:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972166.088079:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972166.088084:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800. 00000400:00000200:3.0:1788972166.088090:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972166.088095:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972166.088100:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084770 00000400:00000010:3.0:1788972166.088102:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084770. 00000100:00000200:3.0:1788972166.088117:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb5180 x1875872179082240/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972166.088190:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972166.088193:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55300. 00000400:00000200:3.0:1788972166.088196:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972166.088200:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972166.088202:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084330 00000400:00000010:3.0:1788972166.088204:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084330. 00000100:00000200:3.0:1788972166.088208:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6300 x1875872179082368/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972182 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972166.096131:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096142:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972166.096151:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096156:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096161:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096166:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d400 00000400:00000200:3.0:1788972166.096172:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096178:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x16a5 [1] + 224 00000400:00000200:3.0:1788972166.096189:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972166.096194:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972166.096209:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb5180 x1875872179082240/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972182 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972166.096252:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096271:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972166.096276:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096279:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096282:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096285:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d480 00000400:00000200:3.0:1788972166.096289:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972166.096293:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x16ad [1] + 224 00000400:00000200:3.0:1788972166.096329:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972166.096333:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972166.096340:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6300 x1875872179082368/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972182 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972166.097165:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163dc0. 00000400:00000200:0.0:1788972166.097172:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:0.0:1788972166.097175:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972166.097183:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6300 x1875872179082368/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972182 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972166.097194:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6300 x1875872179082368/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972182 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972166.097225:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163268. 00000400:00000200:0.0:1788972166.097228:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:0.0:1788972166.097230:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972166.097235:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb5180 x1875872179082240/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972182 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972166.097243:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb5180 x1875872179082240/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972182 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972166.097257:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166be00. 02000000:00000010:0.0:1788972166.097263:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:0.0:1788972166.097268:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6300. 02000000:00000010:0.0:1788972166.097273:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a000. 02000000:00000010:0.0:1788972166.097275:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:0.0:1788972166.097278:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb5180. 00000020:00000010:1.0:1788972166.663497:0:114600:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972166.663586:0:114600:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:2.0:1788972166.791443:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969540. 00000100:00000010:3.0:1788972166.791444:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969ae0. 00000020:00000010:2.0:1788972167.707775:0:114605:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972167.707886:0:114605:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:0.0:1788972168.765464:0:114609:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972168.765582:0:114609:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972169.813095:0:114613:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972169.813181:0:114613:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:2.0:1788972170.845181:0:114617:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972170.845267:0:114617:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:2.0:1788972171.207571:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972171.207596:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1f00. 00000100:00000010:2.0:1788972171.207642:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788972171.207648:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1300. 02000000:00000010:2.0:1788972171.207681:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2e00. 00000400:00000200:2.0:1788972171.207690:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972171.207694:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972171.207700:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972171.207706:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082624, portal 10 00000400:00000010:2.0:1788972171.207713:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0198. 00000100:00000200:2.0:1788972171.207716:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179082624, offset 0 00000400:00000200:2.0:1788972171.207724:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972171.207732:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972171.207739:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972171.207745:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972171.207749:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972171.207753:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972171.207755:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972171.207773:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [431:431:256:4294967295] 192.168.203.124@tcp LPNI seq info [431:431:8:4294967295] 00000400:00000200:2.0:1788972171.207778:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972171.207787:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972171.207793:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972171.207797:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1d00. 00000800:00000200:2.0:1788972171.207804:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972171.207811:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972171.207815:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972171.207830:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972171.207835:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972171.207844:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f72a0. 02000000:00000010:2.0:1788972171.207867:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:2.0:1788972171.207871:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972171.207875:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3370. 00000400:00000010:2.0:1788972171.207878:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0cc0. 00000100:00000200:2.0:1788972171.207881:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082496, portal 25 00000400:00000010:2.0:1788972171.207884:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972171.207886:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179082496, offset 0 00000400:00000200:2.0:1788972171.207890:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972171.207893:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972171.207897:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972171.207900:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972171.207902:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972171.207977:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972171.207980:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972171.207983:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [432:432:255:4294967295] 192.168.203.124@tcp LPNI seq info [432:432:7:4294967295] 00000400:00000200:2.0:1788972171.207987:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972171.207993:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972171.207997:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972171.208000:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1600. 00000800:00000200:2.0:1788972171.208009:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972171.208014:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972171.208016:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972171.208020:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972171.208066:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972171.208070:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1d00. 00000400:00000200:2.0:1788972171.208077:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972171.208081:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972171.208086:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0198 00000400:00000010:2.0:1788972171.208088:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0198. 00000100:00000200:2.0:1788972171.208096:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179082624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972171.208769:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972171.208773:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1600. 00000400:00000200:2.0:1788972171.208777:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972171.208782:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972171.208785:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972171.208787:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972171.208793:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179082496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972187 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972171.212446:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212469:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972171.212478:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212483:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212489:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212493:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d500 00000400:00000200:3.0:1788972171.212499:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212505:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x16c5 [1] + 224 00000400:00000200:3.0:1788972171.212516:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972171.212520:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972171.212534:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179082496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972187 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972171.212569:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212573:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972171.212577:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212580:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212583:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212586:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d580 00000400:00000200:3.0:1788972171.212601:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972171.212606:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x16b5 [1] + 224 00000400:00000200:3.0:1788972171.212615:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972171.212617:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972171.212624:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179082624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972187 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972171.213336:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3370. 00000400:00000200:2.0:1788972171.213343:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0cc0 00000400:00000010:2.0:1788972171.213346:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0cc0. 00000100:00000200:2.0:1788972171.213354:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179082496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972187 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972171.213363:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179082496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972187 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972171.213399:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:2.0:1788972171.213402:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:2.0:1788972171.213405:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972171.213409:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179082624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972187 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972171.213427:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179082624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972187 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972171.213448:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:2.0:1788972171.213469:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1f00. 00000100:00000010:2.0:1788972171.213475:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972171.213479:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2e00. 02000000:00000010:2.0:1788972171.213482:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1300. 00000100:00000010:2.0:1788972171.213484:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 00000020:00000010:1.0:1788972171.871655:0:114621:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972171.872139:0:114621:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:2.0:1788972171.911601:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000100:00000010:3.0:1788972171.911755:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f72a0. 00000020:00000010:3.0:1788972172.924204:0:114625:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972172.924292:0:114625:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:1.0:1788972173.989743:0:114629:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972173.989837:0:114629:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972175.025658:0:114634:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972175.025753:0:114634:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972176.069833:0:114638:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972176.069976:0:114638:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:3.0:1788972176.327474:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5500. 02000000:00000010:3.0:1788972176.327497:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 00000100:00000010:3.0:1788972176.327530:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6d80. 02000000:00000010:3.0:1788972176.327535:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d900. 02000000:00000010:2.0:1788972176.327612:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2800. 00000400:00000200:2.0:1788972176.327622:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972176.327626:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972176.327633:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972176.327641:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082752, portal 25 00000400:00000010:2.0:1788972176.327645:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0cc0. 00000100:00000200:2.0:1788972176.327649:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179082752, offset 0 00000400:00000200:2.0:1788972176.327657:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972176.327666:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972176.327673:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972176.327678:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972176.327682:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972176.327686:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972176.327688:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972176.327692:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [433:433:256:4294967295] 192.168.203.124@tcp LPNI seq info [433:433:8:4294967295] 00000400:00000200:2.0:1788972176.327696:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972176.327705:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972176.327711:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972176.327727:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1300. 00000800:00000200:2.0:1788972176.327734:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972176.327756:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972176.327761:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972176.327779:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972176.327785:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972176.327794:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840. 02000000:00000010:2.0:1788972176.327817:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3600. 00000400:00000200:2.0:1788972176.327822:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972176.327825:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3370. 00000400:00000010:2.0:1788972176.327828:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972176.327832:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179082880, portal 10 00000400:00000010:2.0:1788972176.327836:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0198. 00000100:00000200:2.0:1788972176.327838:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179082880, offset 0 00000400:00000200:2.0:1788972176.327843:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972176.327846:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972176.327849:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972176.327852:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972176.327855:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972176.327857:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972176.327859:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972176.327861:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [434:434:255:4294967295] 192.168.203.124@tcp LPNI seq info [434:434:7:4294967295] 00000400:00000200:2.0:1788972176.327865:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972176.328060:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972176.328064:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1300. 00000400:00000200:3.0:1788972176.328069:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972176.328074:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972176.328092:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972176.328098:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972176.328101:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1f00. 00000800:00000200:2.0:1788972176.328105:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972176.328109:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972176.328112:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972176.328117:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.333234:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0cc0 00000400:00000010:3.0:1788972176.333243:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0cc0. 00000100:00000200:3.0:1788972176.333258:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5500 x1875872179082752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972176.333314:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.333320:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972176.333326:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.333330:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.333335:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.333338:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d600 00000400:00000200:3.0:1788972176.333343:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.333349:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x16d5 [1] + 224 00000400:00000200:3.0:1788972176.333371:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972176.333388:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972176.333395:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5500 x1875872179082752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972192 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972176.333490:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972176.333494:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1f00. 00000400:00000200:3.0:1788972176.333498:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972176.333501:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972176.333504:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0198 00000400:00000010:3.0:1788972176.333507:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0198. 00000100:00000200:3.0:1788972176.333512:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6d80 x1875872179082880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972192 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972176.333615:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:2.0:1788972176.333620:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:2.0:1788972176.333622:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972176.333633:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5500 x1875872179082752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972192 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972176.333643:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5500 x1875872179082752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972192 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972176.333667:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2800. 02000000:00000010:2.0:1788972176.333672:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:2.0:1788972176.333677:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5500. 00000400:00000200:3.0:1788972176.335224:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.335234:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972176.335243:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.335248:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.335254:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.335258:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d680 00000400:00000200:3.0:1788972176.335263:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972176.335271:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x16e5 [1] + 224 00000400:00000200:3.0:1788972176.335286:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972176.335291:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972176.335444:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6d80 x1875872179082880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972192 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972176.337891:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3370. 00000400:00000200:2.0:1788972176.337908:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972176.337911:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972176.337926:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6d80 x1875872179082880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972192 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972176.337939:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6d80 x1875872179082880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972192 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972176.337982:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3600. 02000000:00000010:2.0:1788972176.337990:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d900. 00000100:00000010:2.0:1788972176.337997:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6d80. 00000100:00000010:3.0:1788972177.031385:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840. 00000100:00000010:2.0:1788972177.031528:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000020:00000010:1.0:1788972177.136506:0:114642:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972177.136635:0:114642:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:0.0:1788972178.192774:0:114646:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972178.192865:0:114646:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:3.0:1788972179.221643:0:114650:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972179.221717:0:114650:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972180.251900:0:114654:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972180.251986:0:114654:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972181.288206:0:114658:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972181.288291:0:114658:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000100:00000010:0.0:1788972181.448457:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972181.448476:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55c00. 02000000:00000010:1.0:1788972181.448859:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4400. 00000400:00000200:1.0:1788972181.448869:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972181.448873:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1058. 00000400:00000010:1.0:1788972181.448879:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788972181.448886:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083008, portal 25 00000400:00000010:1.0:1788972181.448891:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788972181.448894:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179083008, offset 0 00000400:00000200:1.0:1788972181.448902:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972181.448911:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972181.448919:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972181.448924:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972181.448928:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972181.448931:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972181.448933:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972181.448937:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [435:435:256:4294967295] 192.168.203.124@tcp LPNI seq info [435:435:8:4294967295] 00000400:00000200:1.0:1788972181.448955:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972181.448965:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972181.448972:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972181.448977:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972181.448984:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972181.448992:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972181.448997:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972181.449013:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972181.449018:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9900. 00000100:00000010:1.0:1788972181.449029:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9c00. 00000100:00000010:0.0:1788972181.449074:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972181.449087:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 02000000:00000010:1.0:1788972181.449365:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4800. 00000400:00000200:1.0:1788972181.449371:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972181.449374:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972181.449378:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588c38. 00000100:00000200:1.0:1788972181.449382:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083136, portal 10 00000400:00000010:1.0:1788972181.449387:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788972181.449389:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179083136, offset 0 00000400:00000200:1.0:1788972181.449393:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972181.449397:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972181.449401:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972181.449403:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972181.449406:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972181.449408:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972181.449411:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972181.449414:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [436:436:255:4294967295] 192.168.203.124@tcp LPNI seq info [436:436:7:4294967295] 00000400:00000200:1.0:1788972181.449418:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972181.449424:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972181.449429:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972181.449432:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972181.449437:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972181.449447:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972181.449450:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972181.449456:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972181.450014:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972181.450019:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972181.450025:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972181.450032:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972181.450037:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445882a8 00000400:00000010:2.0:1788972181.450040:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445882a8. 00000100:00000200:2.0:1788972181.450048:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179083008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972181.450331:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972181.450335:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:2.0:1788972181.450339:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972181.450343:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972181.450353:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:2.0:1788972181.450355:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:2.0:1788972181.450361:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179083136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972197 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972181.455870:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455880:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972181.455890:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455895:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455900:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455904:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d700 00000400:00000200:2.0:1788972181.455909:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455915:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x16f5 [1] + 224 00000400:00000200:2.0:1788972181.455927:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972181.455931:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972181.455948:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179083008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972197 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972181.455987:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455991:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972181.455995:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455997:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.455999:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.456002:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d780 00000400:00000200:2.0:1788972181.456005:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972181.456009:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x16fd [1] + 224 00000400:00000200:2.0:1788972181.456018:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972181.456021:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972181.456027:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179083136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972197 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972181.456413:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972181.456419:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588c38 00000400:00000010:1.0:1788972181.456422:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588c38. 00000100:00000200:1.0:1788972181.456429:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179083136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972197 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972181.456439:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179083136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972197 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972181.456483:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1058. 00000400:00000200:1.0:1788972181.456486:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588660 00000400:00000010:1.0:1788972181.456488:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788972181.456500:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179083008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972197 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972181.456509:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179083008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972197 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972181.456524:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4800. 02000000:00000010:1.0:1788972181.456530:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972181.456536:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 02000000:00000010:1.0:1788972181.456540:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4400. 02000000:00000010:1.0:1788972181.456542:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55c00. 00000100:00000010:1.0:1788972181.456545:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 00000100:00000010:3.0:1788972182.151954:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9c00. 00000100:00000010:2.0:1788972182.152035:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9900. 00000020:00000010:0.0:1788972182.348228:0:114663:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972182.348384:0:114663:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:2.0:1788972183.404165:0:114667:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972183.404252:0:114667:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972184.445229:0:114671:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972184.445622:0:114671:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972185.494802:0:114675:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972185.497723:0:114675:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:1.0:1788972186.531853:0:114679:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972186.531970:0:114679:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:0.0:1788972186.567722:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972186.567747:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55c00. 00000100:00000010:0.0:1788972186.567787:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972186.567793:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 02000000:00000010:0.0:1788972186.567830:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166be00. 00000400:00000200:0.0:1788972186.567842:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972186.567847:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163268. 00000400:00000010:0.0:1788972186.567856:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972186.567865:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083264, portal 25 00000400:00000010:0.0:1788972186.567870:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972186.567874:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179083264, offset 0 00000400:00000200:0.0:1788972186.567885:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972186.567897:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972186.567907:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972186.567914:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972186.567919:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972186.567923:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972186.567926:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972186.567930:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [437:437:256:4294967295] 192.168.203.124@tcp LPNI seq info [437:437:8:4294967295] 00000400:00000200:0.0:1788972186.567935:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972186.567947:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972186.567954:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972186.567960:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55300. 00000800:00000200:0.0:1788972186.567968:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972186.567977:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972186.567982:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55300 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:1.0:1788972186.568058:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5200. 00000400:00000200:1.0:1788972186.568065:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972186.568068:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1058. 00000400:00000010:1.0:1788972186.568073:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788972186.568089:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083392, portal 10 00000400:00000010:1.0:1788972186.568103:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588c38. 00000100:00000200:1.0:1788972186.568106:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179083392, offset 0 00000400:00000200:1.0:1788972186.568111:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972186.568115:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972186.568120:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972186.568122:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972186.568125:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972186.568127:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972186.568129:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972186.568132:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [438:438:255:4294967295] 192.168.203.124@tcp LPNI seq info [438:438:7:4294967295] 00000400:00000200:1.0:1788972186.568136:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972186.568141:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972186.568145:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972186.568150:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972186.568153:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972186.568157:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972186.568160:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972186.568184:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972186.568189:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969420. 00000100:00000010:0.0:1788972186.568207:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969180. 00000400:00000200:1.0:1788972186.568920:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972186.571356:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972186.571369:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55300. 00000400:00000200:3.0:1788972186.571380:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972186.571391:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972186.571400:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:3.0:1788972186.571403:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000100:00000200:3.0:1788972186.571418:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179083264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972186.571521:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972186.571525:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:3.0:1788972186.571529:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972186.571532:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972186.571535:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588c38 00000400:00000010:3.0:1788972186.571537:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588c38. 00000100:00000200:3.0:1788972186.571542:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179083392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972202 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972186.573367:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.573379:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972186.573388:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.573393:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.573399:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.573404:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d880 00000400:00000200:2.0:1788972186.573410:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.573418:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x170d [1] + 224 00000400:00000200:2.0:1788972186.573437:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972186.573443:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972186.573458:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179083392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972202 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972186.573595:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1058. 00000400:00000200:0.0:1788972186.573602:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588660 00000400:00000010:0.0:1788972186.573604:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588660. 00000100:00000200:0.0:1788972186.573611:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179083392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972202 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972186.573620:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179083392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972202 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972186.573669:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5200. 02000000:00000010:0.0:1788972186.573676:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:0.0:1788972186.573683:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000400:00000200:2.0:1788972186.582381:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.582394:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972186.582404:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.582410:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.582417:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.582421:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d800 00000400:00000200:2.0:1788972186.582427:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972186.582435:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1705 [1] + 224 00000400:00000200:2.0:1788972186.582452:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972186.582457:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972186.582474:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179083264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972202 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972186.582644:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163268. 00000400:00000200:0.0:1788972186.582650:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:0.0:1788972186.582652:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972186.582658:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179083264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972202 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972186.582664:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179083264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972202 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972186.582692:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166be00. 02000000:00000010:0.0:1788972186.582698:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55c00. 00000100:00000010:0.0:1788972186.582704:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000100:00000010:3.0:1788972187.271449:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969180. 00000100:00000010:2.0:1788972187.271451:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969420. 00000020:00000010:2.0:1788972187.591296:0:114684:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972187.591427:0:114684:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:3.0:1788972188.657934:0:114688:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972188.658036:0:114688:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:2.0:1788972189.694830:0:114692:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972189.694937:0:114692:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:1.0:1788972190.749200:0:114696:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972190.749293:0:114696:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:0.0:1788972191.688159:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972191.688192:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55c00. 00000100:00000010:0.0:1788972191.688231:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972191.688237:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 02000000:00000010:1.0:1788972191.688320:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4a00. 00000400:00000200:1.0:1788972191.688331:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972191.688335:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972191.688343:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 02000000:00000010:0.0:1788972191.688348:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a600. 00000100:00000200:1.0:1788972191.688351:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083520, portal 25 00000400:00000200:0.0:1788972191.688357:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972191.688359:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588b28. 00000400:00000010:0.0:1788972191.688362:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163268. 00000100:00000200:1.0:1788972191.688364:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179083520, offset 0 00000400:00000010:0.0:1788972191.688372:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000400:00000200:1.0:1788972191.688377:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000100:00000200:0.0:1788972191.688378:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083648, portal 10 00000400:00000200:1.0:1788972191.688387:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972191.688396:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000010:0.0:1788972191.688399:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000400:00000200:1.0:1788972191.688404:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:0.0:1788972191.688405:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179083648, offset 0 00000400:00000200:0.0:1788972191.688415:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972191.688420:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972191.688424:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972191.688428:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972191.688431:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [439:439:256:4294967295] 192.168.203.124@tcp LPNI seq info [439:439:8:4294967295] 00000400:00000200:1.0:1788972191.688435:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972191.688445:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972191.688451:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972191.688457:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972191.688464:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972191.688471:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972191.688475:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972191.688495:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972191.688500:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9180. 00000100:00000010:1.0:1788972191.688509:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9ba0. 00000800:00000200:3.0:1788972191.689257:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972191.689263:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:3.0:1788972191.689268:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972191.689273:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972191.689279:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588b28 00000400:00000010:3.0:1788972191.689281:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588b28. 00000100:00000200:3.0:1788972191.689290:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179083520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788972191.691730:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972191.691742:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972191.691747:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972191.691750:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972191.691753:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972191.691756:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972191.691760:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [440:440:256:4294967295] 192.168.203.124@tcp LPNI seq info [440:440:8:4294967295] 00000400:00000200:0.0:1788972191.691764:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972191.691774:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972191.691780:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972191.691788:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800. 00000800:00000200:0.0:1788972191.691794:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972191.691802:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972191.691806:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972191.691833:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972191.692255:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972191.692260:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800. 00000400:00000200:2.0:1788972191.692264:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972191.692269:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972191.692276:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:2.0:1788972191.692278:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:2.0:1788972191.692288:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179083648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972207 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972191.693954:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972191.693962:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972191.693969:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972191.693973:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972191.693993:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972191.693996:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0d900 00000400:00000200:3.0:1788972191.694001:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972191.694007:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1715 [1] + 224 00000400:00000200:3.0:1788972191.694030:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972191.694035:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972191.694048:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179083520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972207 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972191.694328:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972191.694334:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972191.694336:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972191.694343:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179083520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972207 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972191.694351:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179083520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972207 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972191.694374:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4a00. 02000000:00000010:1.0:1788972191.694380:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55c00. 00000100:00000010:1.0:1788972191.694385:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000400:00000200:2.0:1788972191.694566:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972191.694571:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972191.694576:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972191.694590:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972191.694594:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972191.694598:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0d980 00000400:00000200:2.0:1788972191.694601:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972191.694610:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x171d [1] + 224 00000400:00000200:2.0:1788972191.694649:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972191.694652:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972191.694659:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179083648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972207 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972191.694738:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163268. 00000400:00000200:1.0:1788972191.694742:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:1.0:1788972191.694744:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:1.0:1788972191.694750:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179083648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972207 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972191.694758:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179083648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972207 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972191.694784:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a600. 02000000:00000010:1.0:1788972191.694788:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972191.694790:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000020:00000010:2.0:1788972191.790929:0:114700:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972191.791019:0:114700:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:3.0:1788972192.391455:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9ba0. 00000100:00000010:2.0:1788972192.393542:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9180. 00000020:00000010:0.0:1788972192.828764:0:114704:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972192.828927:0:114704:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972193.864404:0:114709:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972193.864503:0:114709:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:3.0:1788972194.906699:0:114713:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972194.906782:0:114713:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:2.0:1788972195.933625:0:114717:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972195.933712:0:114717:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:0.0:1788972196.807463:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788972196.807487:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788972196.807528:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972196.807534:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 02000000:00000010:1.0:1788972196.807738:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5c00. 00000400:00000200:1.0:1788972196.807749:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972196.807753:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972196.807761:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972196.807769:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083776, portal 25 00000400:00000010:1.0:1788972196.807776:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788972196.807779:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179083776, offset 0 00000400:00000200:1.0:1788972196.807789:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972196.807799:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972196.807807:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972196.807813:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972196.807817:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972196.807820:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972196.807824:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972196.807828:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [441:441:256:4294967295] 192.168.203.124@tcp LPNI seq info [441:441:8:4294967295] 00000400:00000200:1.0:1788972196.807833:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972196.807843:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972196.807852:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972196.807857:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33ff00. 00000800:00000200:1.0:1788972196.807864:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972196.807872:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972196.807877:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972196.807898:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972196.807905:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9180. 00000100:00000010:1.0:1788972196.807914:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9ba0. 02000000:00000010:1.0:1788972196.807933:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4c00. 00000400:00000200:1.0:1788972196.807951:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972196.807954:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972196.807958:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588990. 00000100:00000200:1.0:1788972196.807961:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179083904, portal 10 00000400:00000010:1.0:1788972196.807965:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588a18. 00000100:00000200:1.0:1788972196.807967:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179083904, offset 0 00000400:00000200:1.0:1788972196.807972:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972196.807976:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972196.807979:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972196.807982:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972196.807984:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972196.807986:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972196.807989:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972196.807992:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [442:442:255:4294967295] 192.168.203.124@tcp LPNI seq info [442:442:7:4294967295] 00000400:00000200:1.0:1788972196.807996:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972196.808002:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972196.808006:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972196.808011:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f700. 00000800:00000200:1.0:1788972196.808015:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972196.808020:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972196.808023:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972196.808028:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972196.808135:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972196.808140:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33ff00. 00000400:00000200:3.0:1788972196.808147:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972196.808151:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972196.808156:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588f68 00000400:00000010:3.0:1788972196.808159:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588f68. 00000100:00000200:3.0:1788972196.808180:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179083776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972196.808455:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972196.808458:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f700. 00000400:00000200:3.0:1788972196.808461:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972196.808465:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972196.808468:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588a18 00000400:00000010:3.0:1788972196.808470:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588a18. 00000100:00000200:3.0:1788972196.808475:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179083904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972212 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972196.816144:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972196.816154:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972196.816162:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972196.816177:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972196.816182:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972196.816186:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0da80 00000400:00000200:2.0:1788972196.816192:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972196.816198:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x172d [1] + 224 00000400:00000200:2.0:1788972196.816223:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972196.816228:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972196.816241:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179083904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972212 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972196.816488:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972196.816494:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588990 00000400:00000010:1.0:1788972196.816496:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588990. 00000100:00000200:1.0:1788972196.816503:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179083904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972212 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972196.816512:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179083904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972212 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972196.816542:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4c00. 02000000:00000010:1.0:1788972196.816563:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788972196.816567:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000400:00000200:3.0:1788972196.817418:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972196.817424:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972196.817430:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972196.817434:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972196.817447:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972196.817451:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0da00 00000400:00000200:3.0:1788972196.817456:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972196.817461:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1725 [1] + 224 00000400:00000200:3.0:1788972196.817472:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972196.817476:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972196.817484:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179083776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972212 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972196.817572:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972196.817582:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972196.817584:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972196.817590:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179083776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972212 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972196.817599:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179083776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972212 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972196.817616:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5c00. 02000000:00000010:1.0:1788972196.817620:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:1.0:1788972196.817623:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000020:00000010:2.0:1788972196.980282:0:114721:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972196.980394:0:114721:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:3.0:1788972197.511462:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9ba0. 00000100:00000010:2.0:1788972197.511462:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9180. 00000020:00000010:2.0:1788972198.016849:0:114725:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972198.016923:0:114725:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:1.0:1788972199.049632:0:114729:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972199.049982:0:114729:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972200.078681:0:114733:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972200.078900:0:114733:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:3.0:1788972201.120720:0:114738:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972201.120804:0:114738:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000100:00000010:3.0:1788972201.930310:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4a80. 02000000:00000010:3.0:1788972201.930332:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 02000000:00000010:2.0:1788972201.930429:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:2.0:1788972201.930440:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972201.930444:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3370. 00000400:00000010:2.0:1788972201.930451:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972201.930457:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084032, portal 25 00000400:00000010:2.0:1788972201.930462:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972201.930464:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179084032, offset 0 00000400:00000200:2.0:1788972201.930473:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972201.930482:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972201.930490:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972201.930496:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972201.930501:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972201.930504:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972201.930519:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972201.930523:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [443:443:256:4294967295] 192.168.203.124@tcp LPNI seq info [443:443:8:4294967295] 00000400:00000200:2.0:1788972201.930528:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972201.930537:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972201.930543:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972201.930548:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1600. 00000800:00000200:2.0:1788972201.930555:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972201.930561:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972201.930567:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972201.930582:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972201.930587:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972201.930597:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0. 00000100:00000010:3.0:1788972201.930633:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6300. 02000000:00000010:3.0:1788972201.930639:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 02000000:00000010:2.0:1788972201.930798:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:2.0:1788972201.930804:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972201.930807:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972201.930810:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972201.930813:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084160, portal 10 00000400:00000010:2.0:1788972201.930818:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972201.930820:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179084160, offset 0 00000400:00000200:2.0:1788972201.930824:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972201.930828:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972201.930831:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972201.930834:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972201.930837:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972201.930839:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972201.930841:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972201.930844:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [444:444:255:4294967295] 192.168.203.124@tcp LPNI seq info [444:444:7:4294967295] 00000400:00000200:2.0:1788972201.930847:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972201.930853:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972201.930857:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972201.930867:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1d00. 00000800:00000200:2.0:1788972201.930871:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972201.930875:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972201.930878:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972201.930884:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972201.930928:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972201.931054:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1600. 00000400:00000200:2.0:1788972201.931062:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972201.931067:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972201.931072:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:2.0:1788972201.931074:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972201.931083:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4a80 x1875872179084032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972201.931162:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972201.931165:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1d00. 00000400:00000200:2.0:1788972201.931168:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972201.931181:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972201.931184:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972201.931186:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972201.931191:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6300 x1875872179084160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972217 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972201.934657:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934666:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972201.934675:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934679:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934684:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934688:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0db00 00000400:00000200:3.0:1788972201.934694:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934700:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1735 [1] + 224 00000400:00000200:3.0:1788972201.934711:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972201.934715:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972201.934728:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4a80 x1875872179084032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972217 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972201.934766:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934770:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972201.934774:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934790:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934793:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934795:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0db80 00000400:00000200:3.0:1788972201.934800:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972201.934804:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1745 [1] + 224 00000400:00000200:3.0:1788972201.934822:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972201.934825:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972201.934832:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6300 x1875872179084160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972217 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972201.934981:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:2.0:1788972201.934987:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:2.0:1788972201.935112:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972201.935122:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6300 x1875872179084160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972201.935131:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6300 x1875872179084160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972217 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972201.935160:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3370. 00000400:00000200:2.0:1788972201.935163:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972201.935165:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972201.935188:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4a80 x1875872179084032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972217 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972201.935197:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4a80 x1875872179084032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972217 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972201.935214:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:2.0:1788972201.935221:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:2.0:1788972201.935227:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6300. 02000000:00000010:2.0:1788972201.935231:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:2.0:1788972201.935234:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:2.0:1788972201.935236:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4a80. 00000020:00000010:2.0:1788972202.171590:0:114742:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972202.171848:0:114742:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:2.0:1788972202.633474:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000100:00000010:3.0:1788972202.638219:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0. 00000020:00000010:0.0:1788972203.211739:0:114746:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972203.211828:0:114746:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:2.0:1788972204.254850:0:114750:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972204.254919:0:114750:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:0.0:1788972205.284567:0:114754:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972205.284647:0:114754:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972206.323248:0:114758:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972206.323372:0:114758:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000100:00000010:2.0:1788972207.047545:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788972207.047566:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1d00. 02000000:00000010:3.0:1788972207.047637:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97c00. 00000400:00000200:3.0:1788972207.047647:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972207.047651:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e785d8. 00000400:00000010:3.0:1788972207.047657:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972207.047664:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084288, portal 25 00000400:00000010:3.0:1788972207.047669:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788972207.047672:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179084288, offset 0 00000400:00000200:3.0:1788972207.047680:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972207.047688:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972207.047704:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972207.047709:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972207.047713:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972207.047717:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972207.047720:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972207.047724:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [445:445:256:4294967295] 192.168.203.124@tcp LPNI seq info [445:445:8:4294967295] 00000400:00000200:3.0:1788972207.047798:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972207.047809:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972207.047816:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972207.047821:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00. 00000800:00000200:3.0:1788972207.047827:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972207.047848:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972207.047853:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972207.047871:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972207.047876:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91540. 00000100:00000010:3.0:1788972207.047886:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91360. 00000100:00000010:2.0:1788972207.047978:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972207.047985:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1600. 02000000:00000010:3.0:1788972207.048619:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97400. 00000400:00000200:3.0:1788972207.048626:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972207.048631:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78478. 00000400:00000010:3.0:1788972207.048636:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972207.048640:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084416, portal 10 00000400:00000010:3.0:1788972207.048646:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081b28. 00000100:00000200:3.0:1788972207.048648:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179084416, offset 0 00000400:00000200:3.0:1788972207.048653:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972207.048659:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972207.048663:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972207.048667:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972207.048670:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972207.048672:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972207.048675:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972207.048678:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [446:446:255:4294967295] 192.168.203.124@tcp LPNI seq info [446:446:7:4294967295] 00000400:00000200:3.0:1788972207.048682:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972207.048689:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972207.048902:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972207.048907:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00. 00000400:00000200:2.0:1788972207.048912:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972207.048916:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972207.049442:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:2.0:1788972207.049446:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:2.0:1788972207.049455:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179084288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972207.049584:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972207.049591:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:3.0:1788972207.049598:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972207.049603:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972207.049607:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972207.049630:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972207.050496:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972207.050502:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:2.0:1788972207.050507:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972207.050513:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972207.050531:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081b28 00000400:00000010:2.0:1788972207.050533:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081b28. 00000100:00000200:2.0:1788972207.050541:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179084416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972223 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972207.055198:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.055208:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972207.055217:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.055226:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.055231:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.055235:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0dc00 00000400:00000200:2.0:1788972207.055241:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.055246:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1755 [1] + 224 00000400:00000200:2.0:1788972207.055265:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972207.055270:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972207.055291:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179084288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972223 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972207.057084:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e785d8. 00000400:00000200:3.0:1788972207.057093:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972207.057107:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972207.057131:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179084288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972223 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972207.057158:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179084288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972223 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972207.057186:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97c00. 02000000:00000010:3.0:1788972207.057192:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1d00. 00000100:00000010:3.0:1788972207.057198:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 00000400:00000200:2.0:1788972207.057588:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.057593:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972207.057600:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.057604:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.057609:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.057613:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0dc80 00000400:00000200:2.0:1788972207.057618:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972207.057623:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1765 [1] + 224 00000400:00000200:2.0:1788972207.057637:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972207.057641:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972207.057651:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179084416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972223 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972207.057858:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78478. 00000400:00000200:3.0:1788972207.057863:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788972207.057865:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972207.057871:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179084416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972223 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972207.057880:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179084416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972223 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972207.057900:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97400. 02000000:00000010:3.0:1788972207.057905:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1600. 00000100:00000010:3.0:1788972207.057908:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 00000020:00000010:0.0:1788972207.358232:0:114762:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972207.358768:0:114762:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000100:00000010:3.0:1788972207.751493:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91360. 00000100:00000010:2.0:1788972207.753366:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91540. 00000020:00000010:0.0:1788972208.378728:0:114767:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972208.378822:0:114767:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:3.0:1788972209.398987:0:114771:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972209.399076:0:114771:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:2.0:1788972210.425678:0:114775:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972210.425772:0:114775:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:1.0:1788972211.455177:0:114779:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972211.455411:0:114779:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:3.0:1788972212.167472:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972212.167496:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788972212.167531:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480. 02000000:00000010:3.0:1788972212.167537:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200. 02000000:00000010:2.0:1788972212.167621:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:2.0:1788972212.167632:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972212.167645:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3370. 00000400:00000010:2.0:1788972212.167653:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972212.167660:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084544, portal 25 00000400:00000010:2.0:1788972212.167665:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972212.167668:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179084544, offset 0 00000400:00000200:2.0:1788972212.167676:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972212.167683:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.167691:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972212.167695:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972212.167699:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972212.167703:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972212.167706:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972212.167710:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [447:447:256:4294967295] 192.168.203.124@tcp LPNI seq info [447:447:8:4294967295] 00000400:00000200:2.0:1788972212.167713:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.167721:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972212.167727:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972212.167732:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1900. 00000800:00000200:2.0:1788972212.167738:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972212.167744:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972212.167757:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972212.167874:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972212.167880:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972212.167890:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7d20. 02000000:00000010:2.0:1788972212.167908:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:2.0:1788972212.167913:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972212.167916:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972212.167919:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972212.167936:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084672, portal 10 00000400:00000010:2.0:1788972212.167941:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972212.167944:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179084672, offset 0 00000400:00000200:2.0:1788972212.167949:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972212.167953:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.167957:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972212.167960:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972212.167962:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972212.167964:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972212.167967:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972212.167970:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [448:448:255:4294967295] 192.168.203.124@tcp LPNI seq info [448:448:7:4294967295] 00000400:00000200:2.0:1788972212.167973:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.167979:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972212.167983:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972212.167987:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100. 00000800:00000200:2.0:1788972212.167990:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972212.167994:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972212.168003:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972212.168008:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972212.170905:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972212.170914:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1900. 00000400:00000200:3.0:1788972212.170923:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972212.170931:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972212.170939:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:3.0:1788972212.170942:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:3.0:1788972212.170966:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5880 x1875872179084544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972212.171350:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972212.171354:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100. 00000400:00000200:3.0:1788972212.171358:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972212.171362:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972212.171366:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:3.0:1788972212.171368:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:3.0:1788972212.171375:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7480 x1875872179084672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972228 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972212.173348:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.173358:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972212.173366:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.173370:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.173375:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.173379:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0dd00 00000400:00000200:2.0:1788972212.173383:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972212.173390:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1775 [1] + 224 00000400:00000200:2.0:1788972212.173403:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972212.173407:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972212.173418:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5880 x1875872179084544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972228 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972212.173690:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3370. 00000400:00000200:3.0:1788972212.173696:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:3.0:1788972212.173699:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:3.0:1788972212.173706:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5880 x1875872179084544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972228 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972212.173717:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5880 x1875872179084544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972228 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972212.173741:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:3.0:1788972212.173746:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788972212.173751:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880. 00000400:00000200:3.0:1788972212.174162:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972212.174167:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972212.174173:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972212.174176:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972212.174179:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972212.174183:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0dd80 00000400:00000200:3.0:1788972212.174187:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972212.174192:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1785 [1] + 224 00000400:00000200:3.0:1788972212.174448:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972212.174453:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972212.174475:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7480 x1875872179084672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972228 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972212.174740:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:2.0:1788972212.174754:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972212.174756:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972212.174763:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7480 x1875872179084672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972228 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972212.174782:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7480 x1875872179084672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972228 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972212.174804:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:2.0:1788972212.174809:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200. 00000100:00000010:2.0:1788972212.174813:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480. 00000020:00000010:0.0:1788972212.482700:0:114783:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972212.483549:0:114783:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000100:00000010:2.0:1788972212.871503:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000100:00000010:3.0:1788972212.871505:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7d20. 00000020:00000010:0.0:1788972213.530047:0:114787:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972213.530663:0:114787:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972214.561740:0:114791:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972214.561826:0:114791:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:1.0:1788972215.612591:0:114796:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972215.612670:0:114796:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972216.650967:0:114800:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972216.651066:0:114800:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:1.0:1788972217.287723:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972217.287745:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972217.287783:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972217.287788:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 02000000:00000010:1.0:1788972217.287822:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5a00. 00000400:00000200:1.0:1788972217.287831:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972217.287836:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972217.287842:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972217.287849:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084928, portal 10 00000400:00000010:1.0:1788972217.287870:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588990. 00000100:00000200:1.0:1788972217.287875:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179084928, offset 0 00000400:00000200:1.0:1788972217.287883:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972217.287891:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972217.287898:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972217.287903:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972217.287906:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972217.287910:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972217.287913:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972217.287916:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [449:449:256:4294967295] 192.168.203.124@tcp LPNI seq info [449:449:8:4294967295] 00000400:00000200:1.0:1788972217.287921:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972217.287930:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972217.287936:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972217.287941:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972217.287948:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972217.287956:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972217.287960:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972217.287986:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972217.287992:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9900. 00000100:00000010:1.0:1788972217.288005:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9c00. 02000000:00000010:1.0:1788972217.288034:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5200. 00000400:00000200:1.0:1788972217.288038:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972217.288040:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972217.288044:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588d48. 00000100:00000200:1.0:1788972217.288047:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179084800, portal 25 00000400:00000010:1.0:1788972217.288050:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788972217.288052:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179084800, offset 0 00000400:00000200:1.0:1788972217.288057:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972217.288060:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972217.288073:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972217.288076:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972217.288084:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972217.288087:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972217.288089:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972217.288092:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [450:450:255:4294967295] 192.168.203.124@tcp LPNI seq info [450:450:7:4294967295] 00000400:00000200:1.0:1788972217.288096:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972217.288102:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972217.288106:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972217.288109:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972217.288113:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972217.288117:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972217.288119:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972217.288124:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972217.288339:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972217.288345:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972217.288350:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972217.288355:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972217.288359:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588990 00000400:00000010:2.0:1788972217.288362:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588990. 00000100:00000200:2.0:1788972217.288370:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179084928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972217.288443:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972217.288446:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:2.0:1788972217.288449:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972217.288453:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972217.288455:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:2.0:1788972217.288457:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:2.0:1788972217.288462:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4e00 x1875872179084800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972233 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972217.294156:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294166:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972217.294174:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294179:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294184:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294188:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0de00 00000400:00000200:3.0:1788972217.294193:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294200:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x179d [1] + 224 00000400:00000200:3.0:1788972217.294211:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972217.294215:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972217.294229:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4e00 x1875872179084800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972233 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972217.294266:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294270:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972217.294274:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294276:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294279:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294281:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0de80 00000400:00000200:3.0:1788972217.294284:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972217.294288:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1795 [1] + 224 00000400:00000200:3.0:1788972217.294296:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972217.294326:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972217.294332:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179084928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972233 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972217.295396:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972217.295406:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588d48 00000400:00000010:1.0:1788972217.295409:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588d48. 00000100:00000200:1.0:1788972217.295419:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4e00 x1875872179084800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972233 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972217.295441:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4e00 x1875872179084800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972233 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972217.295464:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972217.295467:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972217.295469:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972217.295473:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179084928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972233 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972217.295482:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179084928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972233 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972217.295506:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5200. 02000000:00000010:1.0:1788972217.295513:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972217.295517:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972217.295537:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5a00. 02000000:00000010:1.0:1788972217.295540:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972217.295542:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000020:00000010:1.0:1788972217.689178:0:114804:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972217.689265:0:114804:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000100:00000010:2.0:1788972217.991435:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9900. 00000100:00000010:3.0:1788972217.991617:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9c00. 00000020:00000010:1.0:1788972218.724241:0:114808:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a3085a0. 00000020:00000010:1.0:1788972218.724596:0:114808:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a3085a0. 00000020:00000010:0.0:1788972219.760633:0:114812:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972219.760727:0:114812:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000020:00000010:2.0:1788972220.784700:0:114816:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972220.784795:0:114816:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000020:00000010:0.0:1788972221.828803:0:114820:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94730. 00000020:00000010:0.0:1788972221.830274:0:114820:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94730. 00000100:00000010:3.0:1788972222.407977:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972222.408005:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788972222.408044:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4000. 02000000:00000010:3.0:1788972222.408049:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:2.0:1788972222.408293:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3a00. 00000400:00000200:2.0:1788972222.408328:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972222.408333:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972222.408340:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972222.408347:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085184, portal 10 00000400:00000010:2.0:1788972222.408354:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972222.408358:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179085184, offset 0 00000400:00000200:2.0:1788972222.408366:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972222.408374:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.408381:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972222.408394:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972222.408397:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972222.408401:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972222.408404:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972222.408407:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [451:451:256:4294967295] 192.168.203.124@tcp LPNI seq info [451:451:8:4294967295] 00000400:00000200:2.0:1788972222.408412:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.408420:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972222.408425:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972222.408431:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1700. 00000800:00000200:2.0:1788972222.408437:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972222.408443:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972222.408448:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972222.408462:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972222.408467:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972222.408477:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0. 02000000:00000010:3.0:1788972222.409520:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97c00. 00000400:00000200:3.0:1788972222.409531:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972222.409534:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78478. 00000400:00000010:3.0:1788972222.409540:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972222.409546:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085056, portal 25 00000400:00000010:3.0:1788972222.409560:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972222.409564:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179085056, offset 0 00000400:00000200:3.0:1788972222.409571:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972222.409577:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.409584:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972222.409588:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972222.409591:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972222.409594:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972222.409597:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000800:00000200:2.0:1788972222.409597:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1788972222.409603:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [452:452:255:4294967295] 192.168.203.124@tcp LPNI seq info [452:452:7:4294967295] 00000800:00000010:2.0:1788972222.409605:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1700. 00000400:00000200:3.0:1788972222.409610:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.409618:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972222.409623:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972222.409623:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000010:3.0:1788972222.409631:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000400:00000200:2.0:1788972222.409633:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000800:00000200:3.0:1788972222.409638:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000400:00000200:2.0:1788972222.409640:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000800:00000200:3.0:1788972222.409645:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972222.409649:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:2.0:1788972222.409655:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972222.409665:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4000 x1875872179085184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972222.409695:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972222.409698:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:2.0:1788972222.409702:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972222.409706:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972222.409709:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:2.0:1788972222.409711:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:2.0:1788972222.409716:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5880 x1875872179085056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972238 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972222.409737:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.412053:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.412062:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972222.412383:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.412880:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.412886:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.412904:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0df00 00000400:00000200:3.0:1788972222.412909:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972222.412916:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x17b5 [1] + 224 00000400:00000200:3.0:1788972222.412933:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972222.412937:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972222.412950:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5880 x1875872179085056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972238 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972222.416406:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78478. 00000400:00000200:2.0:1788972222.416417:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:2.0:1788972222.416420:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:2.0:1788972222.416432:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5880 x1875872179085056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972238 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972222.416443:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5880 x1875872179085056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972238 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972222.416470:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97c00. 02000000:00000010:2.0:1788972222.416477:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:2.0:1788972222.416481:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880. 00000400:00000200:2.0:1788972222.416536:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.416542:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972222.416549:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.416553:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.416558:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.416561:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0df80 00000400:00000200:2.0:1788972222.416566:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972222.416571:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x17a5 [1] + 224 00000400:00000200:2.0:1788972222.416583:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972222.416601:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972222.416610:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4000 x1875872179085184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972238 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972222.417644:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:3.0:1788972222.417653:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:3.0:1788972222.417655:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:3.0:1788972222.417664:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4000 x1875872179085184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972238 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972222.417674:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4000 x1875872179085184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972238 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972222.417697:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3a00. 02000000:00000010:3.0:1788972222.417704:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:3.0:1788972222.417708:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4000. 00000020:00000010:2.0:1788972222.869481:0:114825:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972222.869548:0:114825:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:3.0:1788972223.111402:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0. 00000100:00000010:2.0:1788972223.111402:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000020:00000010:3.0:1788972223.899407:0:114829:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccacc0. 00000020:00000010:3.0:1788972223.899493:0:114829:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccacc0. 00000020:00000010:2.0:1788972223.916828:0:114830:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461661a0. 00000020:00000010:2.0:1788972223.916949:0:114830:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461661a0. 00000100:00000010:1.0:1788972227.527480:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972227.527512:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972227.527561:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972227.527568:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 02000000:00000010:1.0:1788972227.527610:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972227.527623:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972227.527627:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972227.527636:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972227.527645:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085440, portal 10 00000400:00000010:1.0:1788972227.527653:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588d48. 00000100:00000200:1.0:1788972227.527659:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179085440, offset 0 00000400:00000200:1.0:1788972227.527670:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972227.527682:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972227.527692:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972227.527699:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972227.527704:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972227.527708:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972227.527711:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972227.527715:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [453:453:256:4294967295] 192.168.203.124@tcp LPNI seq info [453:453:8:4294967295] 00000400:00000200:1.0:1788972227.527720:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972227.527731:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972227.527738:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972227.527745:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972227.527753:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972227.527782:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972227.527787:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972227.527810:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972227.527816:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d91e0. 00000100:00000010:1.0:1788972227.527838:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9f60. 02000000:00000010:1.0:1788972227.527865:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4000. 00000400:00000200:1.0:1788972227.527870:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972227.527873:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972227.527878:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588e58. 00000100:00000200:1.0:1788972227.527882:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085312, portal 25 00000400:00000010:1.0:1788972227.527886:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788972227.527889:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179085312, offset 0 00000400:00000200:1.0:1788972227.527894:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972227.527898:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972227.527903:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972227.527906:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972227.527909:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972227.527911:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972227.527914:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972227.527917:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [454:454:255:4294967295] 192.168.203.124@tcp LPNI seq info [454:454:7:4294967295] 00000400:00000200:1.0:1788972227.527920:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972227.527927:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972227.527931:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972227.527936:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972227.527940:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972227.527944:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972227.527947:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972227.527951:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972227.527968:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972227.527975:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:3.0:1788972227.527983:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972227.527998:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972227.528005:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588d48 00000400:00000010:3.0:1788972227.528009:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588d48. 00000100:00000200:3.0:1788972227.528021:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4e00 x1875872179085440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972227.528054:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972227.528057:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:3.0:1788972227.528065:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972227.528069:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972227.528072:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588ee0 00000400:00000010:3.0:1788972227.528073:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588ee0. 00000100:00000200:3.0:1788972227.528079:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179085312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972243 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972227.540482:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972227.540503:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972227.540514:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972227.540520:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972227.540527:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972227.540532:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e000 00000400:00000200:2.0:1788972227.540538:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972227.540546:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x17cd [1] + 224 00000400:00000200:2.0:1788972227.540565:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972227.540570:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972227.540591:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179085312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972243 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972227.541006:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972227.541014:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588e58 00000400:00000010:1.0:1788972227.541018:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588e58. 00000100:00000200:1.0:1788972227.541044:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179085312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972243 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972227.541055:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179085312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972243 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972227.541089:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4000. 02000000:00000010:1.0:1788972227.541098:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972227.541103:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000400:00000200:3.0:1788972227.552283:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972227.552295:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972227.552337:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972227.552342:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972227.552348:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972227.552353:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e080 00000400:00000200:3.0:1788972227.552359:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972227.552366:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x17c5 [1] + 224 00000400:00000200:3.0:1788972227.552382:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972227.552387:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972227.552405:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4e00 x1875872179085440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972243 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972227.552550:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972227.552558:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972227.552561:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972227.552568:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4e00 x1875872179085440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972243 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972227.552580:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4e00 x1875872179085440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972243 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972227.552619:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972227.552626:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972227.552632:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4e00. 00000100:00000010:3.0:1788972228.231417:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9f60. 00000100:00000010:2.0:1788972228.231417:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d91e0. 00000100:00000010:1.0:1788972232.648569:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972232.648592:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972232.648635:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972232.648640:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 02000000:00000010:1.0:1788972232.648674:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972232.648684:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972232.648687:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972232.648696:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972232.648703:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085696, portal 10 00000400:00000010:1.0:1788972232.648710:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588e58. 00000100:00000200:1.0:1788972232.648714:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179085696, offset 0 00000400:00000200:1.0:1788972232.648721:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972232.648730:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972232.648752:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972232.648757:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972232.648761:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972232.648765:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972232.648767:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972232.648771:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [455:455:256:4294967295] 192.168.203.124@tcp LPNI seq info [455:455:8:4294967295] 00000400:00000200:1.0:1788972232.648780:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972232.648790:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972232.648795:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972232.648801:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972232.648807:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972232.648814:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972232.648819:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972232.648844:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972232.648849:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9780. 00000100:00000010:1.0:1788972232.648863:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9e40. 02000000:00000010:1.0:1788972232.648891:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4000. 00000400:00000200:1.0:1788972232.648896:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972232.648898:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972232.648902:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788972232.648905:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085568, portal 25 00000400:00000010:1.0:1788972232.648908:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588d48. 00000100:00000200:1.0:1788972232.648910:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179085568, offset 0 00000400:00000200:1.0:1788972232.648914:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972232.648918:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972232.648921:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972232.648924:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972232.648926:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972232.648928:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972232.648930:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972232.648933:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [456:456:255:4294967295] 192.168.203.124@tcp LPNI seq info [456:456:7:4294967295] 00000400:00000200:1.0:1788972232.648943:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972232.648950:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972232.648964:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972232.648968:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972232.648972:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972232.648977:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972232.648980:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972232.648985:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972232.649169:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972232.649174:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972232.649181:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972232.649186:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972232.649191:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588e58 00000400:00000010:2.0:1788972232.649194:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588e58. 00000100:00000200:2.0:1788972232.649203:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179085696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972232.649292:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972232.649326:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:2.0:1788972232.649329:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972232.649333:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972232.649336:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588d48 00000400:00000010:2.0:1788972232.649338:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588d48. 00000100:00000200:2.0:1788972232.649343:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4e00 x1875872179085568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972248 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972232.653105:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653115:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972232.653123:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653129:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653133:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653138:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e100 00000400:00000200:3.0:1788972232.653143:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653149:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x17dd [1] + 224 00000400:00000200:3.0:1788972232.653160:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972232.653165:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972232.653179:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4e00 x1875872179085568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972248 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972232.653242:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653246:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972232.653250:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653253:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653255:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653258:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e180 00000400:00000200:3.0:1788972232.653261:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972232.653265:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x17d5 [1] + 224 00000400:00000200:3.0:1788972232.653273:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972232.653276:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972232.653281:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179085696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972248 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972232.653635:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972232.653641:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588ee0 00000400:00000010:1.0:1788972232.653643:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788972232.653650:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4e00 x1875872179085568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972248 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972232.653659:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4e00 x1875872179085568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972248 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972232.653680:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972232.653682:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972232.653683:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972232.653687:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179085696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972248 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972232.653694:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179085696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972248 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972232.653714:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4000. 02000000:00000010:1.0:1788972232.653719:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972232.653724:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972232.653728:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972232.653730:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972232.653732:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000100:00000010:3.0:1788972233.351503:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9e40. 00000100:00000010:2.0:1788972233.352071:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9780. 00000100:00000010:2.0:1788972237.767868:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972237.767890:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1700. 02000000:00000010:3.0:1788972237.767982:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97400. 00000400:00000200:3.0:1788972237.767992:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972237.767996:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e785d8. 00000400:00000010:3.0:1788972237.768002:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788972237.768009:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085824, portal 25 00000400:00000010:3.0:1788972237.768014:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972237.768016:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179085824, offset 0 00000400:00000200:3.0:1788972237.768025:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972237.768034:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.768042:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972237.768047:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972237.768051:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972237.768055:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972237.768057:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972237.768060:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [457:457:256:4294967295] 192.168.203.124@tcp LPNI seq info [457:457:8:4294967295] 00000400:00000200:3.0:1788972237.768064:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.768086:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972237.768092:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972237.768097:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00. 00000800:00000200:3.0:1788972237.768104:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972237.768113:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972237.768117:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972237.768133:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972237.768138:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91360. 00000100:00000010:3.0:1788972237.768148:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f913c0. 00000100:00000010:2.0:1788972237.768183:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972237.768189:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1000. 02000000:00000010:3.0:1788972237.768282:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f97000. 00000400:00000200:3.0:1788972237.768341:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972237.768344:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78c08. 00000400:00000010:3.0:1788972237.768348:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081d48. 00000100:00000200:3.0:1788972237.768351:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179085952, portal 10 00000400:00000010:3.0:1788972237.768357:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788972237.768359:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179085952, offset 0 00000400:00000200:3.0:1788972237.768363:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972237.768367:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.768370:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972237.768373:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972237.768385:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972237.768387:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972237.768389:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972237.768392:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [458:458:255:4294967295] 192.168.203.124@tcp LPNI seq info [458:458:7:4294967295] 00000400:00000200:3.0:1788972237.768396:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.768402:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972237.768407:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972237.768410:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d700. 00000800:00000200:3.0:1788972237.768414:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972237.768418:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972237.768426:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972237.768432:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972237.768490:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972237.768503:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00. 00000400:00000200:3.0:1788972237.768509:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972237.768514:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972237.768519:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:3.0:1788972237.768521:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972237.768530:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179085824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972237.768597:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972237.768600:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d700. 00000400:00000200:3.0:1788972237.768603:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972237.768607:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972237.768609:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:3.0:1788972237.768610:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788972237.768614:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179085952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972253 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972237.774270:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.774285:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972237.774293:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.774298:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.774719:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.774724:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e280 00000400:00000200:3.0:1788972237.774730:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.774736:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x17f5 [1] + 224 00000400:00000200:3.0:1788972237.774751:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972237.774756:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972237.774768:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179085952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972253 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972237.775690:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.775711:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972237.775719:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.775724:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.775729:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.775732:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e200 00000400:00000200:3.0:1788972237.775737:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972237.775743:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x17e5 [1] + 224 00000400:00000200:3.0:1788972237.775755:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972237.775759:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972237.775767:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179085824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972253 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972237.776426:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78c08. 00000400:00000200:2.0:1788972237.776434:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081d48 00000400:00000010:2.0:1788972237.776436:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081d48. 00000100:00000200:2.0:1788972237.776447:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179085952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972253 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972237.776457:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179085952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972253 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972237.776485:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e785d8. 00000400:00000200:2.0:1788972237.776488:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:2.0:1788972237.776489:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:2.0:1788972237.776494:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179085824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972253 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972237.776501:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179085824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972253 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972237.776514:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97000. 02000000:00000010:2.0:1788972237.776520:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1000. 00000100:00000010:2.0:1788972237.776525:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972237.776528:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f97400. 02000000:00000010:2.0:1788972237.776540:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1700. 00000100:00000010:2.0:1788972237.776542:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000100:00000010:2.0:1788972238.471410:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91360. 00000100:00000010:3.0:1788972238.471454:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f913c0. 00000100:00000010:3.0:1788972242.887505:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4000. 02000000:00000010:3.0:1788972242.887527:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 02000000:00000010:2.0:1788972242.887594:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:2.0:1788972242.887606:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972242.887611:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c08. 00000400:00000010:2.0:1788972242.887619:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972242.887627:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086080, portal 25 00000400:00000010:2.0:1788972242.887633:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972242.887636:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179086080, offset 0 00000400:00000200:2.0:1788972242.887644:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972242.887655:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972242.887664:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972242.887669:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972242.887673:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972242.887677:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972242.887680:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972242.887684:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [459:459:256:4294967295] 192.168.203.124@tcp LPNI seq info [459:459:8:4294967295] 00000400:00000200:2.0:1788972242.887689:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972242.887697:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972242.887704:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972242.887709:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1700. 00000800:00000200:2.0:1788972242.887716:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972242.887724:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972242.887729:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972242.887750:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972242.887755:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972242.887766:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840. 00000100:00000010:3.0:1788972242.887797:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788972242.887814:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:2.0:1788972242.887990:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:2.0:1788972242.887996:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972242.888000:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972242.888019:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972242.888022:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086208, portal 10 00000400:00000010:2.0:1788972242.888027:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972242.888028:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179086208, offset 0 00000400:00000200:2.0:1788972242.888033:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972242.888037:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972242.888041:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972242.888044:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972242.888046:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972242.888048:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972242.888050:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972242.888053:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [460:460:255:4294967295] 192.168.203.124@tcp LPNI seq info [460:460:7:4294967295] 00000400:00000200:2.0:1788972242.888057:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972242.888062:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972242.888066:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972242.888070:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1000. 00000800:00000200:2.0:1788972242.888073:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972242.888077:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972242.888080:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972242.888086:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972242.888144:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972242.888148:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1700. 00000400:00000200:2.0:1788972242.888154:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972242.888158:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972242.888164:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:2.0:1788972242.888166:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972242.888174:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4000 x1875872179086080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972242.888252:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972242.888255:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1000. 00000400:00000200:2.0:1788972242.888258:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972242.888268:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972242.888271:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972242.888273:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972242.888278:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179086208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972258 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972242.890879:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.890897:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972242.890906:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.890910:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.890914:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.890918:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e380 00000400:00000200:3.0:1788972242.890924:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.890929:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1815 [1] + 224 00000400:00000200:3.0:1788972242.890939:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972242.890944:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972242.890955:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179086208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972258 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972242.890990:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.891007:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972242.891012:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.891015:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.891017:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.891020:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e300 00000400:00000200:3.0:1788972242.891023:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972242.891027:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1805 [1] + 224 00000400:00000200:3.0:1788972242.891036:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972242.891039:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972242.891045:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4000 x1875872179086080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972258 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972242.891323:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:2.0:1788972242.891329:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:2.0:1788972242.891331:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972242.891337:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179086208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972258 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972242.891346:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179086208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972258 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972242.891374:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c08. 00000400:00000200:2.0:1788972242.891377:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:2.0:1788972242.891378:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972242.891382:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4000 x1875872179086080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972258 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972242.891389:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4000 x1875872179086080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972258 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972242.891401:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:2.0:1788972242.891407:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:2.0:1788972242.891412:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 02000000:00000010:2.0:1788972242.891415:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:2.0:1788972242.891418:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:2.0:1788972242.891421:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4000. 00000100:00000010:3.0:1788972243.591441:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840. 00000100:00000010:2.0:1788972243.593527:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000100:00000010:1.0:1788972248.007586:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972248.007625:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972248.007667:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972248.007673:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 02000000:00000010:1.0:1788972248.007710:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972248.007721:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972248.007725:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972248.007731:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972248.007738:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086464, portal 10 00000400:00000010:1.0:1788972248.007746:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788972248.007749:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179086464, offset 0 00000400:00000200:1.0:1788972248.007757:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972248.007766:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972248.007773:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972248.007778:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972248.007782:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972248.007786:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972248.007788:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972248.007792:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [461:461:256:4294967295] 192.168.203.124@tcp LPNI seq info [461:461:8:4294967295] 00000400:00000200:1.0:1788972248.007796:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972248.007805:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972248.007811:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972248.007817:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972248.007823:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972248.007830:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972248.007834:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972248.007865:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972248.007869:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d99c0. 00000100:00000010:1.0:1788972248.007883:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d97e0. 02000000:00000010:1.0:1788972248.007907:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4000. 00000400:00000200:1.0:1788972248.007912:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972248.007914:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972248.007923:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788972248.007926:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086336, portal 25 00000400:00000010:1.0:1788972248.007929:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588990. 00000100:00000200:1.0:1788972248.007931:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179086336, offset 0 00000400:00000200:1.0:1788972248.007935:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972248.007939:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972248.007943:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972248.007945:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972248.007948:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972248.007950:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972248.007952:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972248.007956:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [462:462:255:4294967295] 192.168.203.124@tcp LPNI seq info [462:462:7:4294967295] 00000400:00000200:1.0:1788972248.007960:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972248.007966:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972248.007970:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972248.007973:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972248.007977:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972248.007981:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972248.007983:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972248.007988:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972248.010817:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972248.010826:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:2.0:1788972248.010836:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972248.010854:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972248.010878:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588ee0 00000400:00000010:2.0:1788972248.010880:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588ee0. 00000100:00000200:2.0:1788972248.010892:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4e00 x1875872179086464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972248.010979:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972248.010981:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972248.010985:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972248.010988:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972248.010991:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588990 00000400:00000010:2.0:1788972248.010993:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588990. 00000100:00000200:2.0:1788972248.010998:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179086336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972264 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972248.015618:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015631:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972248.015654:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015660:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015669:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015674:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e400 00000400:00000200:3.0:1788972248.015680:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015689:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x182d [1] + 224 00000400:00000200:3.0:1788972248.015704:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972248.015709:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972248.015728:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179086336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972264 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972248.015768:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015772:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972248.015776:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015780:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015783:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972248.015785:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e480 00000400:00000200:3.0:1788972248.015818:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972248.015827:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972248.015839:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:1.0:1788972248.015841:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788972248.015866:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179086336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972264 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972248.015877:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179086336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972264 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972248.015906:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4000. 02000000:00000010:1.0:1788972248.015914:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972248.015919:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000400:00000200:3.0:1788972248.016137:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1825 [1] + 224 00000400:00000200:3.0:1788972248.016153:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972248.016156:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972248.016165:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4e00 x1875872179086464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972264 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972248.016626:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972248.016684:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972248.016687:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972248.016695:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4e00 x1875872179086464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972264 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972248.016704:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4e00 x1875872179086464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972264 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972248.016738:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972248.016744:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972248.016748:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4e00. 00000100:00000010:2.0:1788972248.711416:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d99c0. 00000100:00000010:3.0:1788972248.711516:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d97e0. 00000100:00000010:2.0:1788972253.129667:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972253.129691:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1000. 00000100:00000010:2.0:1788972253.129732:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972253.129736:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1700. 02000000:00000010:2.0:1788972253.129770:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:2.0:1788972253.129779:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972253.129783:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c08. 00000400:00000010:2.0:1788972253.129789:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972253.129796:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086720, portal 10 00000400:00000010:2.0:1788972253.129802:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972253.129805:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179086720, offset 0 00000400:00000200:2.0:1788972253.129825:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972253.129834:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972253.129842:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972253.129847:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972253.129850:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972253.129853:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972253.129856:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972253.129860:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [463:463:256:4294967295] 192.168.203.124@tcp LPNI seq info [463:463:8:4294967295] 00000400:00000200:2.0:1788972253.129878:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972253.129888:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972253.129894:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972253.129900:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100. 00000800:00000200:2.0:1788972253.129907:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972253.129914:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972253.129918:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972253.129933:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972253.129938:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7300. 00000100:00000010:2.0:1788972253.129948:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f72a0. 02000000:00000010:2.0:1788972253.129969:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:2.0:1788972253.129974:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972253.129977:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972253.129981:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972253.129983:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086592, portal 25 00000400:00000010:2.0:1788972253.129987:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972253.129989:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179086592, offset 0 00000400:00000200:2.0:1788972253.129993:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972253.129997:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972253.130000:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972253.130003:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972253.130005:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972253.130007:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972253.130009:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972253.130012:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [464:464:255:4294967295] 192.168.203.124@tcp LPNI seq info [464:464:7:4294967295] 00000400:00000200:2.0:1788972253.130016:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972253.130021:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972253.130025:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972253.130028:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1900. 00000800:00000200:2.0:1788972253.130032:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972253.130041:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972253.130043:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972253.130047:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972253.130101:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972253.130108:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100. 00000400:00000200:2.0:1788972253.130113:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972253.130118:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972253.130122:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:2.0:1788972253.130125:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972253.130133:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179086720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972253.130211:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972253.130214:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1900. 00000400:00000200:2.0:1788972253.130217:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972253.130220:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972253.130223:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:2.0:1788972253.130224:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972253.130229:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179086592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972269 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972253.136533:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136543:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972253.136551:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136556:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136562:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136565:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e580 00000400:00000200:3.0:1788972253.136576:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136582:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1835 [1] + 224 00000400:00000200:3.0:1788972253.136594:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972253.136598:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972253.136611:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179086720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972269 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972253.136649:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136652:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972253.136656:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136658:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136661:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136675:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e500 00000400:00000200:3.0:1788972253.136678:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972253.136682:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1845 [1] + 224 00000400:00000200:3.0:1788972253.136690:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972253.136693:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972253.136758:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179086592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972269 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972253.137045:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:2.0:1788972253.137051:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972253.137053:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972253.137060:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179086592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972269 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972253.137069:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179086592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972269 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972253.137091:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c08. 00000400:00000200:2.0:1788972253.137094:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:2.0:1788972253.137096:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972253.137100:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179086720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972269 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972253.137122:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179086720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972269 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972253.137144:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:2.0:1788972253.137150:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1000. 00000100:00000010:2.0:1788972253.137155:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972253.137159:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:2.0:1788972253.137162:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1700. 00000100:00000010:2.0:1788972253.137164:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 00000100:00000010:2.0:1788972253.831425:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7300. 00000100:00000010:3.0:1788972253.831529:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f72a0. 00000100:00000010:1.0:1788972258.247882:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972258.247905:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972258.247950:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972258.247957:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 02000000:00000010:1.0:1788972258.247993:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972258.248003:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972258.248007:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972258.248015:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972258.248021:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086976, portal 10 00000400:00000010:1.0:1788972258.248028:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788972258.248032:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179086976, offset 0 00000400:00000200:1.0:1788972258.248040:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972258.248049:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972258.248057:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972258.248062:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972258.248066:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972258.248070:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972258.248073:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972258.248077:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [465:465:256:4294967295] 192.168.203.124@tcp LPNI seq info [465:465:8:4294967295] 00000400:00000200:1.0:1788972258.248081:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972258.248090:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972258.248110:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972258.248116:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972258.248124:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972258.248132:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972258.248137:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:0.0:1788972258.248148:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548000. 00000400:00000200:0.0:1788972258.248155:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972258.248159:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163dc0. 00000400:00000200:1.0:1788972258.248160:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972258.248165:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000010:1.0:1788972258.248168:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9720. 00000100:00000200:0.0:1788972258.248170:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179086848, portal 25 00000400:00000010:0.0:1788972258.248176:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480845d8. 00000100:00000200:0.0:1788972258.248179:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179086848, offset 0 00000100:00000010:1.0:1788972258.248186:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9120. 00000400:00000200:0.0:1788972258.248186:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972258.248193:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972258.248198:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972258.248200:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972258.248202:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972258.248205:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972258.248208:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972258.248211:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [466:466:255:4294967295] 192.168.203.124@tcp LPNI seq info [466:466:7:4294967295] 00000400:00000200:0.0:1788972258.248215:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972258.248223:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972258.248234:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972258.248239:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000. 00000800:00000200:0.0:1788972258.248243:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972258.248247:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972258.248250:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972258.248255:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972258.248819:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972258.248824:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972258.248830:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972258.248835:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972258.248841:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:2.0:1788972258.248843:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:2.0:1788972258.248852:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179086976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972258.248929:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972258.248931:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000. 00000400:00000200:2.0:1788972258.248934:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972258.248938:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972258.248940:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480845d8 00000400:00000010:2.0:1788972258.248942:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480845d8. 00000100:00000200:2.0:1788972258.248947:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4e00 x1875872179086848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972274 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972258.250373:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250380:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972258.250386:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250391:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250395:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250400:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e680 00000400:00000200:3.0:1788972258.250405:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250410:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1855 [1] + 224 00000400:00000200:3.0:1788972258.250421:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972258.250425:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972258.250435:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179086976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972274 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972258.250725:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250729:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972258.250733:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250735:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250738:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250741:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e600 00000400:00000200:3.0:1788972258.250745:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972258.250749:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x185d [1] + 224 00000400:00000200:3.0:1788972258.250758:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972258.250761:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000010:1.0:1788972258.250762:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972258.250766:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972258.250769:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972258.250787:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179086976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972274 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972258.250792:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4e00 x1875872179086848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972274 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972258.250797:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179086976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972274 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972258.250969:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972258.250976:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972258.250980:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000400:00000010:0.0:1788972258.251262:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163dc0. 00000400:00000200:0.0:1788972258.251267:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:0.0:1788972258.251269:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788972258.251276:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4e00 x1875872179086848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972274 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972258.251285:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4e00 x1875872179086848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972274 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972258.251329:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548000. 02000000:00000010:0.0:1788972258.251333:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:0.0:1788972258.251336:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4e00. 00000100:00000010:2.0:1788972258.951416:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9720. 00000100:00000010:3.0:1788972258.952938:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9120. 00000100:00000010:1.0:1788972263.367470:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972263.367493:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972263.367543:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972263.367548:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33fb00. 02000000:00000010:1.0:1788972263.367688:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972263.367699:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972263.367703:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972263.367710:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972263.367717:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087232, portal 10 00000400:00000010:1.0:1788972263.367723:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588a18. 00000100:00000200:1.0:1788972263.367726:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179087232, offset 0 00000400:00000200:1.0:1788972263.367744:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972263.367765:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972263.367773:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972263.367779:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972263.367782:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972263.367786:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972263.367789:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972263.367793:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [467:467:256:4294967295] 192.168.203.124@tcp LPNI seq info [467:467:8:4294967295] 00000400:00000200:1.0:1788972263.367797:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972263.367805:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972263.367811:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972263.367816:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f700. 00000800:00000200:1.0:1788972263.367823:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972263.367831:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972263.367835:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972263.367848:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972263.367853:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9540. 00000100:00000010:1.0:1788972263.367867:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9300. 02000000:00000010:1.0:1788972263.367887:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4000. 00000400:00000200:1.0:1788972263.367891:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972263.367894:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972263.367908:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788972263.367911:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087104, portal 25 00000400:00000010:1.0:1788972263.367914:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588b28. 00000100:00000200:1.0:1788972263.367916:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179087104, offset 0 00000400:00000200:1.0:1788972263.367920:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972263.367924:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972263.367927:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972263.367930:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972263.367932:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972263.367935:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972263.367937:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972263.367945:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [468:468:255:4294967295] 192.168.203.124@tcp LPNI seq info [468:468:7:4294967295] 00000400:00000200:1.0:1788972263.367950:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972263.367956:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972263.367960:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972263.367963:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972263.367966:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972263.367980:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972263.367983:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972263.367988:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972263.368217:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972263.368222:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f700. 00000400:00000200:2.0:1788972263.368228:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972263.368232:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972263.368243:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588a18 00000400:00000010:2.0:1788972263.368246:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588a18. 00000100:00000200:2.0:1788972263.368262:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4380 x1875872179087232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972263.368379:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972263.368382:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972263.368385:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972263.368389:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972263.368391:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588b28 00000400:00000010:2.0:1788972263.368393:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588b28. 00000100:00000200:2.0:1788972263.368397:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179087104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972279 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972263.377828:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377837:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972263.377845:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377849:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377854:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377857:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e780 00000400:00000200:2.0:1788972263.377862:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377868:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1865 [1] + 224 00000400:00000200:2.0:1788972263.377878:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972263.377882:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972263.377906:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4380 x1875872179087232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972279 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972263.377938:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377942:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972263.377945:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377947:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377950:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377953:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e700 00000400:00000200:2.0:1788972263.377969:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972263.377981:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x186d [1] + 224 00000400:00000200:2.0:1788972263.377991:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972263.377995:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972263.378001:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179087104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972279 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972263.378081:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972263.378102:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588f68 00000400:00000010:1.0:1788972263.378104:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788972263.378111:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179087104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972279 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972263.378119:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179087104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972279 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972263.378141:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972263.378144:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972263.378146:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972263.378149:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4380 x1875872179087232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972279 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972263.378156:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4380 x1875872179087232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972279 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972263.378184:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4000. 02000000:00000010:1.0:1788972263.378191:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972263.378195:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972263.378206:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972263.378209:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33fb00. 00000100:00000010:1.0:1788972263.378211:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4380. 00000100:00000010:2.0:1788972264.071634:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9540. 00000100:00000010:3.0:1788972264.075626:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9300. 00000100:00000010:1.0:1788972268.488799:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972268.488821:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33fb00. 00000100:00000010:1.0:1788972268.488859:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972268.488863:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33ff00. 02000000:00000010:1.0:1788972268.488891:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972268.488899:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972268.488903:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972268.488909:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972268.488915:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087488, portal 10 00000400:00000010:1.0:1788972268.488921:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788972268.488924:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179087488, offset 0 00000400:00000200:1.0:1788972268.488932:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972268.488940:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972268.488948:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972268.488953:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972268.488957:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972268.488960:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972268.488964:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972268.488967:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [469:469:256:4294967295] 192.168.203.124@tcp LPNI seq info [469:469:8:4294967295] 00000400:00000200:1.0:1788972268.488981:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972268.488991:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972268.488998:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972268.489004:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972268.489010:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972268.489017:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972268.489021:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972268.489039:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972268.489044:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9b40. 00000100:00000010:1.0:1788972268.489056:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9600. 02000000:00000010:1.0:1788972268.489079:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4000. 00000400:00000200:1.0:1788972268.489083:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972268.489100:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972268.489104:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788972268.489107:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087360, portal 25 00000400:00000010:1.0:1788972268.489110:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588c38. 00000100:00000200:1.0:1788972268.489113:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179087360, offset 0 00000400:00000200:1.0:1788972268.489117:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972268.489121:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972268.489124:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972268.489127:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972268.489129:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972268.489131:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972268.489134:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972268.489137:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [470:470:255:4294967295] 192.168.203.124@tcp LPNI seq info [470:470:7:4294967295] 00000400:00000200:1.0:1788972268.489141:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972268.489146:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972268.489151:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972268.489154:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f700. 00000800:00000200:1.0:1788972268.489158:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972268.489162:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972268.489165:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972268.489168:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972268.489244:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972268.489250:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:2.0:1788972268.489256:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972268.489262:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972268.489267:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588f68 00000400:00000010:2.0:1788972268.489270:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588f68. 00000100:00000200:2.0:1788972268.489279:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179087488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972268.489379:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972268.489382:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f700. 00000400:00000200:2.0:1788972268.489386:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972268.489389:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972268.489392:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588c38 00000400:00000010:2.0:1788972268.489393:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588c38. 00000100:00000200:2.0:1788972268.489410:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4380 x1875872179087360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972284 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972268.500588:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.500597:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972268.500607:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.500612:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.500617:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.500621:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e800 00000400:00000200:2.0:1788972268.500626:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.500646:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x187d [1] + 224 00000400:00000200:2.0:1788972268.500665:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972268.500669:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972268.500682:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4380 x1875872179087360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972284 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972268.500775:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972268.500780:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588660 00000400:00000010:1.0:1788972268.500783:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788972268.500790:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4380 x1875872179087360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972284 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972268.500800:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4380 x1875872179087360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972284 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972268.500826:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4000. 02000000:00000010:1.0:1788972268.500833:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33fb00. 00000100:00000010:1.0:1788972268.500845:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4380. 00000400:00000200:2.0:1788972268.507784:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.507793:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972268.507801:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.507807:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.507813:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.507817:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e880 00000400:00000200:2.0:1788972268.507822:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972268.507828:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1875 [1] + 224 00000400:00000200:2.0:1788972268.507843:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972268.507848:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972268.507861:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179087488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972284 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972268.507974:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972268.507980:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972268.507983:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972268.507989:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179087488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972284 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972268.507997:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179087488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972284 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972268.508025:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972268.508030:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33ff00. 00000100:00000010:1.0:1788972268.508036:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000100:00000010:2.0:1788972269.191493:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9b40. 00000100:00000010:3.0:1788972269.191838:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9600. 00000100:00000010:3.0:1788972273.607474:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788972273.607496:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d900. 02000000:00000010:2.0:1788972273.607583:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:2.0:1788972273.607593:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972273.607597:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c08. 00000400:00000010:2.0:1788972273.607604:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972273.607610:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087616, portal 25 00000400:00000010:2.0:1788972273.607616:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972273.607619:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179087616, offset 0 00000400:00000200:2.0:1788972273.607627:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972273.607636:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.607644:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972273.607649:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972273.607653:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972273.607684:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972273.607687:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972273.607691:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [471:471:256:4294967295] 192.168.203.124@tcp LPNI seq info [471:471:8:4294967295] 00000400:00000200:2.0:1788972273.607696:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.607705:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972273.607711:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972273.607716:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1700. 00000800:00000200:2.0:1788972273.607723:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972273.607730:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972273.607735:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972273.607750:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972273.607754:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7960. 00000100:00000010:2.0:1788972273.607766:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7780. 00000100:00000010:3.0:1788972273.607925:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788972273.607931:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 02000000:00000010:2.0:1788972273.607985:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:2.0:1788972273.607989:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972273.607992:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b31b8. 00000400:00000010:2.0:1788972273.607995:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972273.607999:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087744, portal 10 00000400:00000010:2.0:1788972273.608004:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972273.608006:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179087744, offset 0 00000400:00000200:2.0:1788972273.608010:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972273.608014:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.608017:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972273.608020:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972273.608022:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972273.608025:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972273.608027:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972273.608030:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [472:472:255:4294967295] 192.168.203.124@tcp LPNI seq info [472:472:7:4294967295] 00000400:00000200:2.0:1788972273.608034:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.608039:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972273.608053:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972273.608057:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1000. 00000800:00000200:2.0:1788972273.608065:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972273.608068:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972273.608071:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972273.608075:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972273.608111:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972273.608114:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1700. 00000400:00000200:2.0:1788972273.608119:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972273.608122:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972273.608126:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972273.608127:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972273.608134:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179087616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972273.608197:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972273.608199:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1000. 00000400:00000200:2.0:1788972273.608201:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972273.608203:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972273.608205:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:2.0:1788972273.608206:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972273.608209:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179087744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972289 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972273.612106:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.612117:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972273.612126:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.612132:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.612142:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.612147:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0e980 00000400:00000200:2.0:1788972273.612152:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.612158:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1895 [1] + 224 00000400:00000200:2.0:1788972273.612183:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972273.612188:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972273.612202:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179087744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972289 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972273.612578:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b31b8. 00000400:00000200:3.0:1788972273.612584:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:3.0:1788972273.612586:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:3.0:1788972273.612593:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179087744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972289 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972273.612603:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179087744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972289 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972273.612631:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:3.0:1788972273.612638:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:3.0:1788972273.612642:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000400:00000200:2.0:1788972273.617969:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.617978:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972273.617986:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.617990:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.617995:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.617998:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0e900 00000400:00000200:2.0:1788972273.618004:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972273.618009:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1885 [1] + 224 00000400:00000200:2.0:1788972273.618035:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972273.618040:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972273.618052:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179087616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972289 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972273.618158:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c08. 00000400:00000200:3.0:1788972273.618173:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:3.0:1788972273.618176:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:3.0:1788972273.618182:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179087616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972289 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972273.618191:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179087616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972289 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972273.618217:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:3.0:1788972273.618224:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d900. 00000100:00000010:3.0:1788972273.618228:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000100:00000010:2.0:1788972274.311440:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7960. 00000100:00000010:3.0:1788972274.311482:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7780. 00000100:00000010:0.0:1788972278.727698:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788972278.727723:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55000. 00000100:00000010:0.0:1788972278.727763:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788972278.727769:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:1.0:1788972278.727918:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972278.727928:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972278.727932:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972278.727938:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972278.727945:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179087872, portal 25 00000400:00000010:1.0:1788972278.727951:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788972278.727954:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179087872, offset 0 00000400:00000200:1.0:1788972278.727962:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972278.727970:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972278.727978:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972278.727982:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972278.727986:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972278.727990:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972278.727993:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972278.727996:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [473:473:256:4294967295] 192.168.203.124@tcp LPNI seq info [473:473:8:4294967295] 00000400:00000200:1.0:1788972278.728000:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972278.728009:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972278.728015:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972278.728020:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33ff00. 00000800:00000200:1.0:1788972278.728040:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972278.728048:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972278.728052:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972278.728077:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972278.728082:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9960. 00000100:00000010:1.0:1788972278.728092:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d93c0. 02000000:00000010:1.0:1788972278.728113:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4000. 00000400:00000200:1.0:1788972278.728117:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972278.728119:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972278.728123:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788972278.728126:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088000, portal 10 00000400:00000010:1.0:1788972278.728131:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788972278.728134:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179088000, offset 0 00000400:00000200:1.0:1788972278.728138:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972278.728142:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972278.728145:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972278.728147:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972278.728150:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972278.728152:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972278.728154:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972278.728157:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [474:474:255:4294967295] 192.168.203.124@tcp LPNI seq info [474:474:7:4294967295] 00000400:00000200:1.0:1788972278.728160:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972278.728165:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972278.728169:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972278.728183:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972278.728188:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972278.728192:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972278.728195:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972278.728200:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972278.728331:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972278.728336:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33ff00. 00000400:00000200:2.0:1788972278.728342:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972278.728347:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972278.728352:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588660 00000400:00000010:2.0:1788972278.728354:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588660. 00000100:00000200:2.0:1788972278.728363:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179087872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972278.728438:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972278.728441:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:2.0:1788972278.728444:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972278.728456:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972278.728459:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445882a8 00000400:00000010:2.0:1788972278.728468:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445882a8. 00000100:00000200:2.0:1788972278.728474:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179088000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972294 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972278.731571:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.731581:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972278.731588:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.731593:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.731598:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.731602:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ea80 00000400:00000200:2.0:1788972278.731607:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.731613:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x18ad [1] + 224 00000400:00000200:2.0:1788972278.731626:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972278.731639:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972278.731652:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179088000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972294 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972278.733189:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972278.733199:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:1.0:1788972278.733216:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788972278.733229:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179088000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972294 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972278.733239:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179088000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972294 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972278.733271:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4000. 02000000:00000010:1.0:1788972278.733278:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:1.0:1788972278.733283:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000400:00000200:2.0:1788972278.741798:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.741809:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972278.741820:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.741824:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.741830:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.741834:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ea00 00000400:00000200:2.0:1788972278.741840:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972278.741845:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x18a5 [1] + 224 00000400:00000200:2.0:1788972278.741863:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972278.741867:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972278.741880:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179087872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972294 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972278.742142:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972278.742148:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972278.742151:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972278.742157:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179087872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972294 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972278.742166:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179087872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972294 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972278.742191:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972278.742198:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55000. 00000100:00000010:1.0:1788972278.742202:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000100:00000010:3.0:1788972279.431865:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d93c0. 00000100:00000010:2.0:1788972279.433094:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9960. 00000100:00000010:3.0:1788972283.851110:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788972283.851144:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d900. 00000100:00000010:3.0:1788972283.851195:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788972283.851221:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 02000000:00000010:3.0:1788972283.851264:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96e00. 00000400:00000200:3.0:1788972283.851277:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972283.851283:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e781b8. 00000400:00000010:3.0:1788972283.851291:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788972283.851345:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088256, portal 10 00000400:00000010:3.0:1788972283.851355:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972283.851358:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179088256, offset 0 00000400:00000200:3.0:1788972283.851370:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972283.851382:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972283.851392:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972283.851399:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972283.851403:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972283.851408:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972283.851411:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972283.851415:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [475:475:256:4294967295] 192.168.203.124@tcp LPNI seq info [475:475:8:4294967295] 00000400:00000200:3.0:1788972283.851420:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972283.851431:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972283.851439:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972283.851444:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500. 00000800:00000200:3.0:1788972283.851462:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972283.851473:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972283.851478:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972283.851496:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972283.851503:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91660. 00000100:00000010:3.0:1788972283.851521:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91900. 02000000:00000010:3.0:1788972283.851800:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96c00. 00000400:00000200:3.0:1788972283.851807:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972283.851810:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78000. 00000400:00000010:3.0:1788972283.851816:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788972283.851819:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088128, portal 25 00000400:00000010:3.0:1788972283.851831:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081d48. 00000100:00000200:3.0:1788972283.851833:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179088128, offset 0 00000400:00000200:3.0:1788972283.851838:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972283.851842:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972283.851846:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972283.851850:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972283.851852:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972283.851855:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972283.851858:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972283.851861:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [476:476:255:4294967295] 192.168.203.124@tcp LPNI seq info [476:476:7:4294967295] 00000400:00000200:3.0:1788972283.851864:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972283.851870:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972283.851874:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972283.851878:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:3.0:1788972283.851882:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972283.851887:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972283.851890:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972283.851894:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972283.851958:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972283.851963:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500. 00000400:00000200:3.0:1788972283.851970:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972283.851975:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972283.851982:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:3.0:1788972283.851985:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972283.851996:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179088256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972283.852084:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972283.852088:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:3.0:1788972283.852091:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972283.852095:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972283.852097:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081d48 00000400:00000010:3.0:1788972283.852690:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081d48. 00000100:00000200:3.0:1788972283.852700:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179088128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972299 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972283.864604:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864639:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972283.864651:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864656:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864662:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864666:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0eb00 00000400:00000200:2.0:1788972283.864672:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864680:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x18c5 [1] + 224 00000400:00000200:2.0:1788972283.864693:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972283.864697:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972283.864715:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179088128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972299 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972283.864765:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864768:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972283.864772:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864774:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864777:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864779:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0eb80 00000400:00000200:2.0:1788972283.864783:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972283.864787:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x18b5 [1] + 224 00000400:00000200:2.0:1788972283.864799:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972283.864803:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972283.864808:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179088256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972299 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972283.864830:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78000. 00000400:00000200:3.0:1788972283.864839:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:3.0:1788972283.864843:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788972283.864851:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179088128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972299 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972283.864862:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179088128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972299 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972283.864895:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e781b8. 00000400:00000200:3.0:1788972283.864898:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:3.0:1788972283.864900:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788972283.864903:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179088256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972299 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972283.864910:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179088256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972299 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972283.864934:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96c00. 02000000:00000010:3.0:1788972283.864941:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d900. 00000100:00000010:3.0:1788972283.864957:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788972283.864962:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96e00. 02000000:00000010:3.0:1788972283.864964:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:3.0:1788972283.864966:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000100:00000010:2.0:1788972284.551478:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91660. 00000100:00000010:3.0:1788972284.555242:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91900. 00000100:00000010:0.0:1788972288.967467:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788972288.967489:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788972288.967532:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788972288.967537:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 02000000:00000010:0.0:1788972288.967595:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548600. 00000400:00000200:0.0:1788972288.967605:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972288.967614:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163dc0. 00000400:00000010:0.0:1788972288.967621:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788972288.967628:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088512, portal 10 00000400:00000010:0.0:1788972288.967634:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084f68. 00000100:00000200:0.0:1788972288.967652:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179088512, offset 0 00000400:00000200:0.0:1788972288.967661:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972288.967668:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972288.967676:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972288.967681:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972288.967684:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 02000000:00000010:1.0:1788972288.967792:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:1.0:1788972288.967798:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972288.967801:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972288.967805:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972288.967808:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088384, portal 25 00000400:00000010:1.0:1788972288.967811:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788972288.967813:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179088384, offset 0 00000400:00000200:1.0:1788972288.967826:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972288.968396:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972288.968400:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972288.968404:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [477:477:256:4294967295] 192.168.203.124@tcp LPNI seq info [477:477:8:4294967295] 00000400:00000200:0.0:1788972288.968413:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972288.968424:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972288.968431:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972288.968437:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55c00. 00000800:00000200:0.0:1788972288.968444:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972288.968451:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972288.968460:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972288.968490:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972288.968497:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449695a0. 00000100:00000010:0.0:1788972288.968578:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969a20. 00000800:00000200:3.0:1788972288.968614:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972288.968620:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55c00. 00000400:00000200:3.0:1788972288.968625:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972288.968630:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972288.968635:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084f68 00000400:00000010:3.0:1788972288.968638:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084f68. 00000100:00000200:3.0:1788972288.968646:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179088512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972288.968722:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972288.968728:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972288.968732:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972288.968735:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972288.968737:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972288.968746:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972288.968749:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [478:478:256:4294967295] 192.168.203.124@tcp LPNI seq info [478:478:8:4294967295] 00000400:00000200:1.0:1788972288.968753:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972288.968759:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972288.968763:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972288.968768:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972288.968772:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972288.968777:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972288.968780:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972288.968859:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972288.968903:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972288.968907:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:3.0:1788972288.968910:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972288.968914:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972288.968917:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:3.0:1788972288.968919:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:3.0:1788972288.969274:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179088384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972304 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972288.972694:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972704:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972288.972713:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972718:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972723:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972727:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ec80 00000400:00000200:3.0:1788972288.972732:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972739:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x18d5 [1] + 224 00000400:00000200:3.0:1788972288.972750:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972288.972755:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972288.972769:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179088512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972304 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972288.972808:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972812:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972288.972816:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972819:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972822:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972825:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ec00 00000400:00000200:3.0:1788972288.972828:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972288.972833:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x18dd [1] + 224 00000400:00000200:3.0:1788972288.972842:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972288.972858:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972288.972865:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179088384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972304 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972288.972961:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972288.972967:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972288.972969:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972288.972977:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179088384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972304 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972288.972986:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179088384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972304 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972288.973021:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:1.0:1788972288.973028:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788972288.973033:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000400:00000010:0.0:1788972288.973350:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163dc0. 00000400:00000200:0.0:1788972288.973356:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:0.0:1788972288.973358:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788972288.973366:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179088512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972304 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972288.973376:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179088512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972304 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972288.973397:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548600. 02000000:00000010:0.0:1788972288.973402:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:0.0:1788972288.973510:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000100:00000010:2.0:1788972289.671832:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449695a0. 00000100:00000010:3.0:1788972289.677580:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969a20. 00000001:02000400:1.0:1788972293.053774:0:115091:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_304: @@@@@@ FAIL: Failed to unload modules before test execution 00000100:00000010:1.0:1788972294.091890:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972294.091912:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f300. 00000100:00000010:1.0:1788972294.091941:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972294.091946:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4a33f700. 02000000:00000010:1.0:1788972294.091980:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50f40e00. 00000400:00000200:1.0:1788972294.091990:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972294.091995:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972294.092002:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972294.092016:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088640, portal 25 00000400:00000010:1.0:1788972294.092022:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588aa0. 00000100:00000200:1.0:1788972294.092025:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179088640, offset 0 00000400:00000200:1.0:1788972294.092034:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972294.092042:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972294.092050:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972294.092054:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972294.092058:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972294.092062:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972294.092064:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972294.092068:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [479:479:256:4294967295] 192.168.203.124@tcp LPNI seq info [479:479:8:4294967295] 00000400:00000200:1.0:1788972294.092072:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972294.092080:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972294.092086:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972294.092089:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33f800. 00000800:00000200:1.0:1788972294.092096:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972294.092102:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972294.092107:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972294.092136:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972294.092141:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9c60. 00000100:00000010:1.0:1788972294.092150:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9960. 02000000:00000010:1.0:1788972294.092169:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50f41800. 00000400:00000200:1.0:1788972294.092174:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972294.092176:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972294.092179:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445886e8. 00000100:00000200:1.0:1788972294.092183:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179088768, portal 10 00000400:00000010:1.0:1788972294.092188:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788972294.092190:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179088768, offset 0 00000400:00000200:1.0:1788972294.092195:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972294.092198:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972294.092201:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972294.092210:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972294.092213:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972294.092215:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972294.092217:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972294.092219:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [480:480:255:4294967295] 192.168.203.124@tcp LPNI seq info [480:480:7:4294967295] 00000400:00000200:1.0:1788972294.092223:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972294.092229:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972294.092233:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972294.092237:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4a33fb00. 00000800:00000200:1.0:1788972294.092241:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972294.092245:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972294.092247:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4a33fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972294.092252:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972294.092646:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972294.092651:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33f800. 00000400:00000200:3.0:1788972294.092657:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972294.092662:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972294.092667:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588aa0 00000400:00000010:3.0:1788972294.092670:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588aa0. 00000100:00000200:3.0:1788972294.092678:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179088640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972294.092745:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972294.092747:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4a33fb00. 00000400:00000200:3.0:1788972294.092751:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972294.092755:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972294.092757:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:3.0:1788972294.092759:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:3.0:1788972294.092764:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4380 x1875872179088768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972310 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972294.102989:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.102999:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972294.103008:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103014:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103019:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103032:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0ed80 00000400:00000200:2.0:1788972294.103037:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103043:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x18ed [1] + 224 00000400:00000200:2.0:1788972294.103064:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972294.103076:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972294.103088:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4380 x1875872179088768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972310 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972294.103131:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103134:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972294.103138:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103140:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103143:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103146:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ed00 00000400:00000200:2.0:1788972294.103150:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972294.103154:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x18e5 [1] + 224 00000400:00000200:2.0:1788972294.103161:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972294.103164:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972294.103170:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179088640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972310 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972294.103979:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972294.103986:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445886e8 00000400:00000010:1.0:1788972294.103989:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445886e8. 00000100:00000200:1.0:1788972294.103996:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4380 x1875872179088768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972310 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972294.104006:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4380 x1875872179088768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972310 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972294.104033:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972294.104035:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972294.104037:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972294.104041:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179088640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972310 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972294.104114:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179088640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972310 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972294.104262:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50f41800. 02000000:00000010:1.0:1788972294.104268:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f700. 00000100:00000010:1.0:1788972294.104282:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972294.104287:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50f40e00. 02000000:00000010:1.0:1788972294.104290:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4a33f300. 00000100:00000010:1.0:1788972294.104292:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. Debug log: 3094 lines, 3094 kept, 0 dropped, 0 bad.