00000100:00000010:3.0F:1788972310.151440:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969120. 00000100:00000010:2.0F:1788972310.151648:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969540. 00000100:00000010:1.0F:1788972314.567542:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972314.567596:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988600. 00000100:00000010:1.0:1788972314.567649:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972314.567656:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988c00. 02000000:00000010:1.0:1788972314.567706:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50d52a00. 00000400:00000200:1.0:1788972314.567719:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972314.567724:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972314.567733:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972314.567743:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179089792, portal 10 00000400:00000010:1.0:1788972314.567753:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445886e8. 00000100:00000200:1.0:1788972314.567758:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179089792, offset 0 00000400:00000200:1.0:1788972314.567770:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972314.567784:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972314.567794:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972314.567800:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972314.567806:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972314.567810:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972314.567814:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972314.567818:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [487:487:256:4294967295] 192.168.203.124@tcp LPNI seq info [487:487:8:4294967295] 00000400:00000200:1.0:1788972314.567823:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972314.567836:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972314.567844:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972314.567853:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988000. 00000800:00000200:1.0:1788972314.567863:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972314.567873:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972314.567878:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972314.567898:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972314.567905:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9900. 00000100:00000010:1.0:1788972314.567918:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9780. 02000000:00000010:1.0:1788972314.567954:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50d52200. 00000400:00000200:1.0:1788972314.567969:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972314.567972:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972314.567983:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588110. 00000100:00000200:1.0:1788972314.567986:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179089664, portal 25 00000400:00000010:1.0:1788972314.567990:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972314.567992:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179089664, offset 0 00000400:00000200:1.0:1788972314.567998:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972314.568002:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972314.568006:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972314.568009:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972314.568011:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972314.568014:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972314.568017:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972314.568019:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [488:488:255:4294967295] 192.168.203.124@tcp LPNI seq info [488:488:7:4294967295] 00000400:00000200:1.0:1788972314.568023:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972314.568029:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972314.568033:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972314.568037:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988500. 00000800:00000200:1.0:1788972314.568041:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972314.568045:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972314.568048:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972314.568052:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972314.573440:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972314.573454:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988000. 00000400:00000200:2.0:1788972314.573466:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972314.573476:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972314.573485:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445886e8 00000400:00000010:2.0:1788972314.573488:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445886e8. 00000100:00000200:2.0:1788972314.573505:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4380 x1875872179089792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972314.573610:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972314.573613:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988500. 00000400:00000200:2.0:1788972314.573616:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972314.573620:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972314.573624:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445883b8 00000400:00000010:2.0:1788972314.573626:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445883b8. 00000100:00000200:2.0:1788972314.573633:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179089664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972330 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972314.582919:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.582933:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972314.582946:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.582952:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.582968:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.582973:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f180 00000400:00000200:2.0:1788972314.582980:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.582988:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1945 [1] + 224 00000400:00000200:2.0:1788972314.583002:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972314.583007:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972314.583026:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4380 x1875872179089792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972330 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972314.583080:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.583084:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972314.583088:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.583108:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.583111:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.583114:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f100 00000400:00000200:2.0:1788972314.583119:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972314.583123:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x194d [1] + 224 00000400:00000200:2.0:1788972314.583136:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972314.583140:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972314.583147:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179089664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972330 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972314.590079:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972314.590097:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588110 00000400:00000010:1.0:1788972314.590100:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588110. 00000100:00000200:1.0:1788972314.590119:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179089664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972314.590132:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179089664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972330 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972314.590165:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972314.590168:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972314.590171:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972314.590178:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4380 x1875872179089792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972330 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972314.590210:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4380 x1875872179089792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972330 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972314.590238:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50d52200. 02000000:00000010:1.0:1788972314.590246:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988600. 00000100:00000010:1.0:1788972314.590252:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972314.590256:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50d52a00. 02000000:00000010:1.0:1788972314.590259:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988c00. 00000100:00000010:1.0:1788972314.590261:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4380. 00000100:00000010:2.0:1788972315.271523:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9900. 00000100:00000010:3.0:1788972315.271634:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9780. 00000100:00000010:1.0:1788972319.687397:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972319.687430:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988c00. 00000100:00000010:1.0:1788972319.687477:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972319.687483:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988600. 02000000:00000010:1.0:1788972319.687528:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a800. 00000400:00000200:1.0:1788972319.687541:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972319.687546:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972319.687554:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972319.687563:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090048, portal 10 00000400:00000010:1.0:1788972319.687572:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588110. 00000100:00000200:1.0:1788972319.687576:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179090048, offset 0 00000400:00000200:1.0:1788972319.687588:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972319.687600:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972319.687610:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972319.687616:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972319.687621:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972319.687626:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972319.687629:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972319.687633:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [489:489:256:4294967295] 192.168.203.124@tcp LPNI seq info [489:489:8:4294967295] 00000400:00000200:1.0:1788972319.687638:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972319.687650:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972319.687679:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972319.687687:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988500. 00000800:00000200:1.0:1788972319.687707:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972319.687717:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972319.687724:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972319.687747:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972319.687753:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9720. 00000100:00000010:1.0:1788972319.687771:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d95a0. 02000000:00000010:0.0F:1788972319.687871:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96800. 00000400:00000200:0.0:1788972319.687884:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972319.687887:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000010:0.0:1788972319.687892:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972319.687895:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179089920, portal 25 00000400:00000010:0.0:1788972319.687899:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480843b8. 00000100:00000200:0.0:1788972319.687902:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179089920, offset 0 00000400:00000200:0.0:1788972319.687907:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972319.687911:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972319.687915:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972319.687918:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972319.687921:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972319.687923:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972319.687925:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972319.687928:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [490:490:255:4294967295] 192.168.203.124@tcp LPNI seq info [490:490:7:4294967295] 00000400:00000200:0.0:1788972319.687932:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972319.687938:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972319.687943:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972319.687947:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55900. 00000800:00000200:0.0:1788972319.687951:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972319.687956:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972319.687959:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972319.687963:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972319.688359:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972319.688365:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988500. 00000400:00000200:2.0:1788972319.688372:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972319.688378:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972319.688385:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588110 00000400:00000010:2.0:1788972319.688388:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588110. 00000100:00000200:2.0:1788972319.688398:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179090048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972319.688497:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972319.688500:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55900. 00000400:00000200:2.0:1788972319.688504:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972319.688507:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972319.688510:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480843b8 00000400:00000010:2.0:1788972319.688512:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480843b8. 00000100:00000200:2.0:1788972319.688517:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4380 x1875872179089920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972335 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972319.696025:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696040:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972319.696050:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696056:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696062:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696081:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f200 00000400:00000200:3.0:1788972319.696088:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696096:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x195d [1] + 224 00000400:00000200:3.0:1788972319.696111:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972319.696116:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972319.696146:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4380 x1875872179089920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972335 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972319.696174:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696178:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972319.696182:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696185:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696188:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696191:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f280 00000400:00000200:3.0:1788972319.696195:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972319.696200:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1955 [1] + 224 00000400:00000200:3.0:1788972319.696209:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972319.696213:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972319.696220:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179090048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972335 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972319.704612:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:0.0:1788972319.704630:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:0.0:1788972319.704634:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972319.704654:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4380 x1875872179089920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972335 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972319.704668:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4380 x1875872179089920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972335 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972319.704718:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96800. 02000000:00000010:0.0:1788972319.704751:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988c00. 00000100:00000010:0.0:1788972319.704759:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4380. 00000400:00000010:1.0:1788972319.706819:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972319.706832:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972319.706835:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972319.706853:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179090048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972335 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972319.706866:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179090048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972335 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972319.706906:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a800. 02000000:00000010:1.0:1788972319.706913:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988600. 00000100:00000010:1.0:1788972319.706920:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000100:00000010:2.0:1788972320.395118:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9720. 00000100:00000010:3.0:1788972320.455577:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d95a0. 00000001:02000400:2.0:1788972323.049020:0:116379:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 310: Set timeout and verify ========== 12:45:21 (1788972321) 00000020:00000010:1.0:1788972323.759588:0:116404:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972323.759663:0:116404:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972323.804710:0:116409:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972323.804909:0:116409:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:0.0:1788972324.811080:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972324.811100:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788972324.811128:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972324.811132:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 02000000:00000010:0.0:1788972324.811281:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50d96800. 00000400:00000200:0.0:1788972324.811290:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972324.811294:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000010:0.0:1788972324.811351:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972324.811358:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090176, portal 25 00000400:00000010:0.0:1788972324.811363:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084440. 00000100:00000200:0.0:1788972324.811366:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179090176, offset 0 00000400:00000200:0.0:1788972324.811374:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972324.811382:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972324.811389:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972324.811394:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972324.811399:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972324.811402:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972324.811405:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972324.811409:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [491:491:256:4294967295] 192.168.203.124@tcp LPNI seq info [491:491:8:4294967295] 00000400:00000200:0.0:1788972324.811413:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972324.811422:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972324.811428:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972324.811432:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000. 00000800:00000200:0.0:1788972324.811438:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972324.811445:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972324.811449:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972324.811470:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972324.811475:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969ae0. 00000100:00000010:0.0:1788972324.811484:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969c60. 02000000:00000010:0.0:1788972324.811515:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb527d1800. 00000400:00000200:0.0:1788972324.811519:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972324.811522:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000010:0.0:1788972324.811525:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972324.811527:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090304, portal 10 00000400:00000010:0.0:1788972324.811532:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084908. 00000100:00000200:0.0:1788972324.811534:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179090304, offset 0 00000400:00000200:0.0:1788972324.811539:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972324.811542:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972324.811545:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972324.811548:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972324.811550:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972324.811552:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972324.811555:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972324.811557:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [492:492:255:4294967295] 192.168.203.124@tcp LPNI seq info [492:492:7:4294967295] 00000400:00000200:0.0:1788972324.811561:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972324.811568:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972324.811571:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972324.811574:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55900. 00000800:00000200:0.0:1788972324.811577:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972324.811581:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972324.811584:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972324.811589:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972324.813138:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972324.813147:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000. 00000400:00000200:2.0:1788972324.813154:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972324.813170:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972324.813176:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084440 00000400:00000010:2.0:1788972324.813178:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084440. 00000100:00000200:2.0:1788972324.813189:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179090176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972324.813265:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972324.813268:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55900. 00000400:00000200:2.0:1788972324.813271:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972324.813274:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972324.813277:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084908 00000400:00000010:2.0:1788972324.813278:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084908. 00000100:00000200:2.0:1788972324.813283:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179090304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972340 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972324.825735:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825746:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972324.825754:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825773:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825779:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825783:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f300 00000400:00000200:3.0:1788972324.825788:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825794:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1965 [1] + 224 00000400:00000200:3.0:1788972324.825805:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972324.825810:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972324.825823:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179090176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972340 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972324.825858:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825861:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972324.825865:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825868:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825871:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825873:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f380 00000400:00000200:3.0:1788972324.825876:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972324.825880:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x196d [1] + 224 00000400:00000200:3.0:1788972324.825888:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972324.825891:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972324.825897:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179090304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972340 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972324.826411:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:0.0:1788972324.826417:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990 00000400:00000010:0.0:1788972324.826420:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972324.826427:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179090304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972340 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972324.826438:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179090304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972340 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972324.826481:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:0.0:1788972324.826484:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:0.0:1788972324.826486:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972324.826491:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179090176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972340 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972324.826499:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179090176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972340 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972324.826513:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb527d1800. 02000000:00000010:0.0:1788972324.826518:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:0.0:1788972324.826523:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972324.826526:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50d96800. 02000000:00000010:0.0:1788972324.826529:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788972324.826531:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000020:00000010:2.0:1788972324.847345:0:116413:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972324.847426:0:116413:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000100:00000010:3.0:1788972325.511445:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969c60. 00000100:00000010:2.0:1788972325.511939:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969ae0. 00000020:00000010:3.0:1788972325.908593:0:116417:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972325.909159:0:116417:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972326.976406:0:116421:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972326.980831:0:116421:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:1.0:1788972328.053550:0:116425:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972328.053625:0:116425:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972329.137526:0:116430:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972329.137609:0:116430:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:0.0:1788972329.927688:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972329.927711:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788972329.927750:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972329.927755:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 02000000:00000010:1.0:1788972329.927881:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb527d1800. 00000400:00000200:1.0:1788972329.927892:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972329.927895:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972329.927901:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972329.927921:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090432, portal 25 00000400:00000010:1.0:1788972329.927927:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788972329.927930:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179090432, offset 0 00000400:00000200:1.0:1788972329.927939:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972329.927947:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972329.927955:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972329.927960:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972329.927964:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972329.927967:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972329.927970:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972329.927975:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [493:493:256:4294967295] 192.168.203.124@tcp LPNI seq info [493:493:8:4294967295] 00000400:00000200:1.0:1788972329.927979:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972329.927988:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972329.927995:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972329.927999:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988600. 00000800:00000200:1.0:1788972329.928006:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972329.928014:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972329.928018:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972329.928040:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972329.928045:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9f60. 00000100:00000010:1.0:1788972329.928057:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9300. 02000000:00000010:1.0:1788972329.928078:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001200. 00000400:00000200:1.0:1788972329.928082:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972329.928085:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972329.928088:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588550. 00000100:00000200:1.0:1788972329.928091:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090560, portal 10 00000400:00000010:1.0:1788972329.928096:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588dd0. 00000100:00000200:1.0:1788972329.928098:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179090560, offset 0 00000400:00000200:1.0:1788972329.928102:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972329.928105:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972329.928108:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972329.928116:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972329.928119:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972329.928121:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972329.928124:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972329.928126:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [494:494:255:4294967295] 192.168.203.124@tcp LPNI seq info [494:494:7:4294967295] 00000400:00000200:1.0:1788972329.928131:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972329.928137:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972329.928141:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972329.928145:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988000. 00000800:00000200:1.0:1788972329.928150:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972329.928155:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972329.928158:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972329.928164:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972329.928358:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972329.928363:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988600. 00000400:00000200:2.0:1788972329.928368:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972329.928373:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972329.928378:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330 00000400:00000010:2.0:1788972329.928380:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330. 00000100:00000200:2.0:1788972329.928388:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179090432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972329.928444:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972329.928447:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988000. 00000400:00000200:2.0:1788972329.928450:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972329.928454:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972329.928456:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588dd0 00000400:00000010:2.0:1788972329.928458:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588dd0. 00000100:00000200:2.0:1788972329.928463:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179090560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972345 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972329.933491:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.933994:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972329.934005:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934010:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934027:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934031:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f400 00000400:00000200:3.0:1788972329.934036:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934042:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1975 [1] + 224 00000400:00000200:3.0:1788972329.934063:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972329.934068:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972329.934081:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179090432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972345 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972329.934121:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934125:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972329.934129:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934132:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934135:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934137:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f480 00000400:00000200:3.0:1788972329.934140:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972329.934144:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x197d [1] + 224 00000400:00000200:3.0:1788972329.934153:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972329.934157:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972329.934163:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179090560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972345 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972329.934255:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972329.934261:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588550 00000400:00000010:1.0:1788972329.934273:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588550. 00000100:00000200:1.0:1788972329.934280:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179090560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972345 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972329.934290:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179090560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972345 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972329.934350:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972329.934354:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972329.934356:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972329.934370:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179090432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972345 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972329.934378:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179090432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972345 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972329.934391:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001200. 02000000:00000010:1.0:1788972329.934397:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:1.0:1788972329.934408:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 02000000:00000010:1.0:1788972329.934413:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb527d1800. 02000000:00000010:1.0:1788972329.934416:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788972329.934418:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000020:00000010:0.0:1788972330.185619:0:116434:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94990. 00000020:00000010:0.0:1788972330.185708:0:116434:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94990. 00000100:00000010:2.0:1788972330.631414:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9f60. 00000100:00000010:3.0:1788972330.631505:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9300. 00000020:00000010:2.0:1788972331.218765:0:116438:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972331.218847:0:116438:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:1.0:1788972332.266236:0:116442:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972332.268479:0:116442:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:3.0:1788972333.284876:0:116446:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972333.284959:0:116446:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:0.0:1788972334.328288:0:116450:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94990. 00000020:00000010:0.0:1788972334.328380:0:116450:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94990. 00000100:00000010:0.0:1788972335.048160:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972335.048181:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788972335.048216:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972335.048221:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 02000000:00000010:1.0:1788972335.048342:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb527d1800. 00000400:00000200:1.0:1788972335.048352:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972335.048356:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972335.048373:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972335.048379:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090688, portal 25 00000400:00000010:1.0:1788972335.048384:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588550. 00000100:00000200:1.0:1788972335.048395:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179090688, offset 0 00000400:00000200:1.0:1788972335.048405:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972335.048413:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972335.048420:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972335.048425:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972335.048429:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972335.048433:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972335.048435:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972335.048439:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [495:495:256:4294967295] 192.168.203.124@tcp LPNI seq info [495:495:8:4294967295] 00000400:00000200:1.0:1788972335.048444:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972335.048453:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972335.048459:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972335.048465:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988000. 00000800:00000200:1.0:1788972335.048471:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972335.048479:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972335.048483:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972335.048503:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972335.048508:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9600. 00000100:00000010:1.0:1788972335.048521:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d97e0. 02000000:00000010:1.0:1788972335.048555:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001200. 00000400:00000200:1.0:1788972335.048560:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972335.048563:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972335.048566:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588000. 00000100:00000200:1.0:1788972335.048569:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090816, portal 10 00000400:00000010:1.0:1788972335.048574:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445885d8. 00000100:00000200:1.0:1788972335.048576:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179090816, offset 0 00000400:00000200:1.0:1788972335.048580:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972335.048583:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972335.048586:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972335.048589:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972335.048592:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972335.048594:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972335.048596:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972335.048599:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [496:496:255:4294967295] 192.168.203.124@tcp LPNI seq info [496:496:7:4294967295] 00000400:00000200:1.0:1788972335.048603:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972335.048609:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972335.048613:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972335.048617:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988600. 00000800:00000200:1.0:1788972335.048621:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972335.048625:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972335.048627:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972335.048633:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972335.048726:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972335.048731:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988000. 00000400:00000200:2.0:1788972335.048737:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972335.048741:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972335.048745:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588550 00000400:00000010:2.0:1788972335.048746:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588550. 00000100:00000200:2.0:1788972335.048754:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179090688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972335.048925:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972335.048929:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988600. 00000400:00000200:2.0:1788972335.048932:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972335.048936:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972335.048938:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445885d8 00000400:00000010:2.0:1788972335.048940:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445885d8. 00000100:00000200:2.0:1788972335.048945:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179090816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972351 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972335.052891:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.052901:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972335.052909:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.052914:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.052919:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.052923:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f580 00000400:00000200:3.0:1788972335.052928:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.053504:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x198d [1] + 224 00000400:00000200:3.0:1788972335.053517:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972335.053522:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972335.053536:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179090816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972351 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972335.053643:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.053647:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972335.053651:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.053654:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.053657:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.053660:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f500 00000400:00000200:3.0:1788972335.053663:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972335.053668:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1985 [1] + 224 00000400:00000200:3.0:1788972335.053676:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972335.053680:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972335.053698:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179090688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972351 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972335.053921:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972335.053927:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588000 00000400:00000010:1.0:1788972335.053929:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588000. 00000100:00000200:1.0:1788972335.053996:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179090816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972351 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972335.054006:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179090816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972351 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972335.054055:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972335.054061:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972335.054063:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972335.054067:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179090688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972351 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972335.054075:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179090688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972351 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972335.054089:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001200. 02000000:00000010:1.0:1788972335.054095:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972335.054100:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 02000000:00000010:1.0:1788972335.054104:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb527d1800. 02000000:00000010:1.0:1788972335.054108:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:1.0:1788972335.054110:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000020:00000010:1.0:1788972335.364575:0:116454:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972335.364658:0:116454:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:2.0:1788972335.753785:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9600. 00000100:00000010:3.0:1788972335.756841:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d97e0. 00000020:00000010:3.0:1788972336.431780:0:116459:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972336.431894:0:116459:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:1.0:1788972337.516784:0:116463:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972337.516866:0:116463:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:0.0:1788972338.575582:0:116467:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94990. 00000020:00000010:0.0:1788972338.576149:0:116467:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94990. 00000020:00000010:1.0:1788972339.640145:0:116471:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972339.640240:0:116471:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:0.0:1788972340.170331:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972340.170353:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788972340.170407:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972340.170411:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 02000000:00000010:1.0:1788972340.170507:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001600. 00000400:00000200:1.0:1788972340.170518:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972340.170522:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972340.170528:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972340.170534:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179090944, portal 25 00000400:00000010:1.0:1788972340.170538:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588000. 00000100:00000200:1.0:1788972340.170542:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179090944, offset 0 00000400:00000200:1.0:1788972340.170549:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972340.170558:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972340.170565:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972340.170570:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972340.170579:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972340.170583:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972340.170585:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972340.170589:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [497:497:256:4294967295] 192.168.203.124@tcp LPNI seq info [497:497:8:4294967295] 00000400:00000200:1.0:1788972340.170593:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972340.170602:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972340.170608:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972340.170612:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988c00. 00000800:00000200:1.0:1788972340.170618:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972340.170626:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972340.170631:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972340.170651:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972340.170655:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9c60. 00000100:00000010:1.0:1788972340.170665:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9960. 02000000:00000010:1.0:1788972340.170683:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001200. 00000400:00000200:1.0:1788972340.170687:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972340.170689:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972340.170692:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588bb0. 00000100:00000200:1.0:1788972340.170694:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091072, portal 10 00000400:00000010:1.0:1788972340.170699:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588088. 00000100:00000200:1.0:1788972340.170701:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179091072, offset 0 00000400:00000200:1.0:1788972340.170705:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972340.170708:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972340.170711:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972340.170714:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972340.170716:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972340.170719:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972340.170721:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972340.170724:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [498:498:255:4294967295] 192.168.203.124@tcp LPNI seq info [498:498:7:4294967295] 00000400:00000200:1.0:1788972340.170727:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972340.170733:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972340.170744:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972340.170747:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988500. 00000800:00000200:1.0:1788972340.170751:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972340.170754:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972340.170757:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972340.170763:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972340.170860:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972340.170864:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988c00. 00000400:00000200:2.0:1788972340.170870:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972340.170875:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972340.170880:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588000 00000400:00000010:2.0:1788972340.170882:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588000. 00000100:00000200:2.0:1788972340.170891:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179090944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972340.170958:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972340.170960:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988500. 00000400:00000200:2.0:1788972340.170968:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972340.170971:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972340.170974:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588088 00000400:00000010:2.0:1788972340.170976:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588088. 00000100:00000200:2.0:1788972340.170981:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179091072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972356 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972340.178059:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178072:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972340.178081:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178087:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178093:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178098:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f680 00000400:00000200:3.0:1788972340.178115:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178123:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x199d [1] + 224 00000400:00000200:3.0:1788972340.178136:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972340.178156:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972340.178175:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179091072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972356 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972340.178221:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178225:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972340.178229:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178232:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178237:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178241:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f600 00000400:00000200:3.0:1788972340.178246:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972340.178254:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1995 [1] + 224 00000400:00000200:3.0:1788972340.178264:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972340.178270:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972340.178278:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179090944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972356 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972340.178285:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972340.178297:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588bb0 00000400:00000010:1.0:1788972340.178339:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588bb0. 00000100:00000200:1.0:1788972340.178358:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179091072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972356 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972340.178372:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179091072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972356 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972340.191013:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972340.191026:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972340.191028:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972340.191042:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179090944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972356 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972340.191055:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179090944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972356 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972340.191136:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001200. 02000000:00000010:1.0:1788972340.191145:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:1.0:1788972340.191149:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 02000000:00000010:1.0:1788972340.191153:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001600. 02000000:00000010:1.0:1788972340.191155:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:1.0:1788972340.191157:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000020:00000010:1.0:1788972340.704801:0:116475:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972340.704985:0:116475:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:3.0:1788972340.871513:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9960. 00000100:00000010:2.0:1788972340.871516:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9c60. 00000020:00000010:1.0:1788972341.740532:0:116479:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972341.740613:0:116479:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:3.0:1788972342.784818:0:116484:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972342.785551:0:116484:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:0.0:1788972343.850992:0:116488:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94990. 00000020:00000010:0.0:1788972343.851232:0:116488:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94990. 00000020:00000010:2.0:1788972344.913012:0:116492:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972344.913119:0:116492:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000100:00000010:0.0:1788972345.287439:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972345.287477:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 00000100:00000010:0.0:1788972345.287530:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972345.287535:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 02000000:00000010:1.0:1788972345.287693:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb449af600. 00000400:00000200:1.0:1788972345.287707:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972345.287712:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972345.287721:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972345.287731:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091200, portal 25 00000400:00000010:1.0:1788972345.287738:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588bb0. 00000100:00000200:1.0:1788972345.287741:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179091200, offset 0 00000400:00000200:1.0:1788972345.287752:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972345.287764:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972345.287774:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972345.287781:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972345.287786:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972345.287790:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972345.287793:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972345.287798:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [499:499:256:4294967295] 192.168.203.124@tcp LPNI seq info [499:499:8:4294967295] 00000400:00000200:1.0:1788972345.287803:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972345.287815:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972345.287823:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972345.287831:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988500. 00000800:00000200:1.0:1788972345.287839:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972345.287848:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972345.287865:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972345.287887:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972345.287894:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d93c0. 00000100:00000010:1.0:1788972345.287907:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9960. 02000000:00000010:1.0:1788972345.287932:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c200. 00000400:00000200:1.0:1788972345.287937:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972345.287940:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972345.287943:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588440. 00000100:00000200:1.0:1788972345.287946:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091328, portal 10 00000400:00000010:1.0:1788972345.287971:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588908. 00000100:00000200:1.0:1788972345.287975:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179091328, offset 0 00000400:00000200:1.0:1788972345.287980:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972345.287984:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972345.287987:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972345.287990:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972345.287993:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972345.287995:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972345.287997:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972345.288000:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [500:500:255:4294967295] 192.168.203.124@tcp LPNI seq info [500:500:7:4294967295] 00000400:00000200:1.0:1788972345.288003:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972345.288009:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972345.288013:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972345.288017:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988c00. 00000800:00000200:1.0:1788972345.288021:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972345.288025:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972345.288028:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972345.288045:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972345.288175:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972345.288181:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988500. 00000400:00000200:2.0:1788972345.288190:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972345.288197:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972345.288204:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588bb0 00000400:00000010:2.0:1788972345.288206:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588bb0. 00000100:00000200:2.0:1788972345.288218:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179091200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972345.288350:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972345.288353:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988c00. 00000400:00000200:2.0:1788972345.288356:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972345.288360:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972345.288363:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588908 00000400:00000010:2.0:1788972345.288365:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588908. 00000100:00000200:2.0:1788972345.288369:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179091328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972361 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972345.296750:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972345.296763:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972345.296774:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972345.296780:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972345.296787:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972345.296793:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f780 00000400:00000200:3.0:1788972345.296799:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972345.296807:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x19ad [1] + 224 00000400:00000200:3.0:1788972345.296824:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972345.296830:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972345.296851:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179091328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972361 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972345.297935:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972345.297945:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588440 00000400:00000010:1.0:1788972345.297948:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588440. 00000100:00000200:1.0:1788972345.297960:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179091328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972361 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972345.297973:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179091328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972361 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972345.298016:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c200. 02000000:00000010:1.0:1788972345.298253:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788972345.298263:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000400:00000200:2.0:1788972345.311720:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972345.311733:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972345.311746:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972345.311752:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972345.311758:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972345.311763:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f700 00000400:00000200:2.0:1788972345.311786:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972345.311794:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x19a5 [1] + 224 00000400:00000200:2.0:1788972345.311822:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972345.311827:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972345.311845:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179091200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972361 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972345.312795:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972345.312802:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972345.312815:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972345.312826:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179091200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972361 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972345.312837:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179091200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972361 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972345.313000:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb449af600. 02000000:00000010:1.0:1788972345.313011:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972345.313016:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 00000020:00000010:3.0:1788972345.982933:0:116496:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972345.983047:0:116496:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000100:00000010:2.0:1788972345.993387:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d93c0. 00000100:00000010:3.0:1788972345.993554:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9960. 00000020:00000010:0.0:1788972347.034098:0:116500:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94990. 00000020:00000010:0.0:1788972347.037893:0:116500:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94990. 00000020:00000010:2.0:1788972348.078897:0:116504:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972348.079623:0:116504:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:3.0:1788972349.132786:0:116508:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972349.132985:0:116508:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972350.174078:0:116513:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972350.174187:0:116513:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000100:00000010:3.0:1788972350.407635:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972350.407669:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972350.407728:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788972350.407734:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:2.0:1788972350.407844:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549200. 00000400:00000200:2.0:1788972350.407857:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972350.407862:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3370. 00000400:00000010:2.0:1788972350.407870:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972350.407880:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091456, portal 25 00000400:00000010:2.0:1788972350.407886:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8. 00000100:00000200:2.0:1788972350.407889:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179091456, offset 0 00000400:00000200:2.0:1788972350.407899:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972350.407984:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.407996:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972350.408003:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972350.408008:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972350.408014:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972350.408016:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972350.408021:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [501:501:256:4294967295] 192.168.203.124@tcp LPNI seq info [501:501:8:4294967295] 00000400:00000200:2.0:1788972350.408025:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.408036:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972350.408045:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972350.408051:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1b00. 00000800:00000200:2.0:1788972350.408059:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972350.408068:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972350.408093:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972350.408160:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972350.408167:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7de0. 00000100:00000010:2.0:1788972350.408181:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7420. 02000000:00000010:2.0:1788972350.408205:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548600. 00000400:00000200:2.0:1788972350.408210:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972350.408213:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3790. 00000400:00000010:2.0:1788972350.408217:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788972350.408221:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091584, portal 10 00000400:00000010:2.0:1788972350.408227:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972350.408229:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179091584, offset 0 00000400:00000200:2.0:1788972350.408233:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972350.408237:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.408241:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972350.408244:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972350.408247:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972350.408249:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972350.408252:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972350.408254:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [502:502:255:4294967295] 192.168.203.124@tcp LPNI seq info [502:502:7:4294967295] 00000400:00000200:2.0:1788972350.408258:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.408264:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972350.408268:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972350.408272:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1300. 00000800:00000200:2.0:1788972350.408276:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972350.408281:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972350.408284:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972350.408290:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972350.414459:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972350.414549:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1b00. 00000400:00000200:2.0:1788972350.414561:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972350.414574:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972350.414604:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8 00000400:00000010:2.0:1788972350.414607:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8. 00000100:00000200:2.0:1788972350.414624:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179091456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972350.419707:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972350.419721:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1300. 00000400:00000200:2.0:1788972350.419734:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972350.419745:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972350.419756:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972350.419759:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972350.419773:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6680 x1875872179091584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972366 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972350.430677:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430745:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972350.430755:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430761:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430767:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430772:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f880 00000400:00000200:2.0:1788972350.430778:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430785:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x19c5 [1] + 224 00000400:00000200:2.0:1788972350.430798:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972350.430803:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972350.430820:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6680 x1875872179091584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972366 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972350.430862:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430866:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972350.430870:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430872:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430874:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430877:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f800 00000400:00000200:2.0:1788972350.430898:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972350.430903:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x19b5 [1] + 224 00000400:00000200:2.0:1788972350.430913:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972350.430916:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972350.430923:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179091456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972366 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972350.431018:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3790. 00000400:00000200:3.0:1788972350.431025:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28 00000400:00000010:3.0:1788972350.431027:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28. 00000100:00000200:3.0:1788972350.431034:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6680 x1875872179091584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972366 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972350.431043:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6680 x1875872179091584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972366 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972350.432246:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3370. 00000400:00000200:3.0:1788972350.432258:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:3.0:1788972350.432261:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:3.0:1788972350.432273:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179091456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972366 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972350.432284:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179091456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972366 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972350.432323:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548600. 02000000:00000010:3.0:1788972350.432330:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:3.0:1788972350.432336:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788972350.432340:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549200. 02000000:00000010:3.0:1788972350.432361:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972350.432364:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000100:00000010:2.0:1788972351.112005:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7de0. 00000100:00000010:3.0:1788972351.119631:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7420. 00000020:00000010:1.0:1788972351.201774:0:116517:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972351.201873:0:116517:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972352.267838:0:116521:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972352.270691:0:116521:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972353.388952:0:116525:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972353.390687:0:116525:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000020:00000010:2.0:1788972354.465254:0:116529:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972354.465446:0:116529:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000100:00000010:3.0:1788972355.533655:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972355.533675:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972355.533713:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788972355.533718:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:3.0:1788972355.533753:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5600. 00000400:00000200:3.0:1788972355.533762:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972355.533766:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78370. 00000400:00000010:3.0:1788972355.533772:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972355.533779:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091712, portal 25 00000400:00000010:3.0:1788972355.533785:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788972355.533788:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179091712, offset 0 00000400:00000200:3.0:1788972355.533796:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972355.533805:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.533811:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972355.533816:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972355.533819:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972355.533823:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972355.533825:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972355.533828:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [503:503:256:4294967295] 192.168.203.124@tcp LPNI seq info [503:503:8:4294967295] 00000400:00000200:3.0:1788972355.533832:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.533841:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972355.533846:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972355.533850:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788972355.533856:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972355.533863:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972355.533868:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972355.533880:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972355.533885:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91240. 00000100:00000010:3.0:1788972355.533906:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91d80. 02000000:00000010:3.0:1788972355.533925:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4e00. 00000400:00000200:3.0:1788972355.533930:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972355.533932:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972355.533935:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972355.533939:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091840, portal 10 00000400:00000010:3.0:1788972355.533944:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972355.533946:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179091840, offset 0 00000400:00000200:3.0:1788972355.533950:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972355.533953:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.533956:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972355.533959:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972355.533961:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972355.533962:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972355.533964:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972355.533967:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [504:504:255:4294967295] 192.168.203.124@tcp LPNI seq info [504:504:7:4294967295] 00000400:00000200:3.0:1788972355.533971:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.533977:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972355.533981:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972355.533984:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:3.0:1788972355.533987:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972355.533991:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972355.533994:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972355.533999:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972355.534058:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972355.534062:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788972355.534067:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972355.534071:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972355.534076:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:3.0:1788972355.534078:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788972355.534096:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179091712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972355.534181:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972355.534185:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:3.0:1788972355.534187:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972355.534191:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972355.534194:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972355.534196:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972355.534201:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6680 x1875872179091840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972371 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972355.549488:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.549497:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972355.549506:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.549510:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.549516:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.549520:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0f980 00000400:00000200:3.0:1788972355.549525:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.549530:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x19e5 [1] + 224 00000400:00000200:3.0:1788972355.549559:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972355.549565:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972355.549580:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6680 x1875872179091840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972371 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972355.549620:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972355.549625:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788972355.549627:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972355.549634:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6680 x1875872179091840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972371 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972355.549643:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6680 x1875872179091840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972371 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972355.549672:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4e00. 02000000:00000010:3.0:1788972355.549689:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:3.0:1788972355.549693:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6680. 00000400:00000200:3.0:1788972355.553209:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.553219:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972355.553227:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.553232:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.553236:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.553240:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0f900 00000400:00000200:3.0:1788972355.553245:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972355.553251:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x19d5 [1] + 224 00000400:00000200:3.0:1788972355.553265:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972355.553271:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972355.553285:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179091712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972371 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972355.553374:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78370. 00000400:00000200:3.0:1788972355.553379:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:3.0:1788972355.553381:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972355.553388:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179091712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972371 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972355.553397:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179091712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972371 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972355.553422:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5600. 02000000:00000010:3.0:1788972355.553428:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972355.553433:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000020:00000010:2.0:1788972355.561047:0:116533:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972355.561155:0:116533:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000100:00000010:3.0:1788972356.232426:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91d80. 00000100:00000010:2.0:1788972356.232536:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91240. 00000020:00000010:0.0:1788972356.650792:0:116537:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972356.650901:0:116537:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972357.697763:0:116542:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972357.697856:0:116542:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000020:00000010:3.0:1788972358.755010:0:116546:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972358.757452:0:116546:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:1.0:1788972359.818136:0:116550:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972359.818226:0:116550:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:0.0:1788972360.647824:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972360.647847:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 00000100:00000010:0.0:1788972360.647887:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788972360.647892:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55900. 02000000:00000010:1.0:1788972360.647982:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c600. 00000400:00000200:1.0:1788972360.647992:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972360.647996:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972360.648002:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972360.648015:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179091968, portal 25 00000400:00000010:1.0:1788972360.648023:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588440. 00000100:00000200:1.0:1788972360.648033:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179091968, offset 0 00000400:00000200:1.0:1788972360.648056:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972360.648065:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972360.648071:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972360.648076:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972360.648080:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972360.648084:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972360.648087:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972360.648091:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [505:505:256:4294967295] 192.168.203.124@tcp LPNI seq info [505:505:8:4294967295] 00000400:00000200:1.0:1788972360.648095:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972360.648126:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972360.648132:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972360.648137:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988600. 00000800:00000200:1.0:1788972360.648144:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972360.648151:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972360.648155:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972360.648175:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972360.648180:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9c60. 00000100:00000010:1.0:1788972360.648190:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d97e0. 00000800:00000200:3.0:1788972360.648651:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972360.648657:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988600. 00000400:00000200:3.0:1788972360.648662:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972360.648677:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972360.648684:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588440 00000400:00000010:3.0:1788972360.648687:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588440. 00000100:00000200:3.0:1788972360.648695:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179091968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972376 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972360.648794:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c400. 00000400:00000200:1.0:1788972360.648800:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972360.648802:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972360.648806:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972360.648809:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092096, portal 10 00000400:00000010:1.0:1788972360.648814:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588cc0. 00000100:00000200:1.0:1788972360.648817:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179092096, offset 0 00000400:00000200:1.0:1788972360.648821:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972360.648825:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972360.648829:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972360.648833:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972360.648836:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972360.648838:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972360.648840:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972360.648843:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [506:506:256:4294967295] 192.168.203.124@tcp LPNI seq info [506:506:8:4294967295] 00000400:00000200:1.0:1788972360.648847:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972360.648853:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972360.648857:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972360.648861:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988000. 00000800:00000200:1.0:1788972360.648866:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972360.648870:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972360.648879:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972360.648899:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972360.648968:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972360.648971:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988000. 00000400:00000200:3.0:1788972360.648974:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972360.648978:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972360.648981:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588cc0 00000400:00000010:3.0:1788972360.648983:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588cc0. 00000100:00000200:3.0:1788972360.648989:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179092096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972376 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972360.651138:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.651147:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972360.651156:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.651160:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.651165:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.651169:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0fa00 00000400:00000200:3.0:1788972360.651174:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.651179:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x19f5 [1] + 224 00000400:00000200:3.0:1788972360.651192:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972360.651196:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972360.651207:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179091968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972376 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972360.651495:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972360.651500:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972360.651503:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972360.651509:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179091968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972376 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972360.651518:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179091968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972376 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972360.651544:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c600. 02000000:00000010:1.0:1788972360.651550:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:1.0:1788972360.651555:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000400:00000200:3.0:1788972360.655451:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.655460:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972360.655468:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.655473:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.655478:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.655495:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0fa80 00000400:00000200:3.0:1788972360.655500:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972360.655506:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x19fd [1] + 224 00000400:00000200:3.0:1788972360.655519:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972360.655523:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972360.655535:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179092096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972376 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972360.657594:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972360.657606:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770 00000400:00000010:1.0:1788972360.657609:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972360.657621:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179092096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972376 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972360.657632:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179092096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972376 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972360.657665:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c400. 02000000:00000010:1.0:1788972360.657674:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55900. 00000100:00000010:1.0:1788972360.657678:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000020:00000010:2.0:1788972360.911645:0:116554:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972360.911734:0:116554:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000100:00000010:3.0:1788972361.351372:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d97e0. 00000100:00000010:2.0:1788972361.351510:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9c60. 00000020:00000010:2.0:1788972361.949025:0:116558:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972361.949132:0:116558:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972363.007382:0:116562:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972363.007466:0:116562:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:0.0:1788972364.066868:0:116566:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972364.066965:0:116566:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000020:00000010:1.0:1788972365.113960:0:116571:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972365.114058:0:116571:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:3.0:1788972365.767530:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972365.767553:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972365.767601:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788972365.767607:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:3.0:1788972365.767639:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4c00. 00000400:00000200:3.0:1788972365.767648:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972365.767652:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78370. 00000400:00000010:3.0:1788972365.767659:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788972365.767666:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092352, portal 10 00000400:00000010:3.0:1788972365.767673:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788972365.767676:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179092352, offset 0 00000400:00000200:3.0:1788972365.767683:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972365.767691:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.767699:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972365.767704:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972365.767708:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972365.767711:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972365.767714:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972365.767717:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [507:507:256:4294967295] 192.168.203.124@tcp LPNI seq info [507:507:8:4294967295] 00000400:00000200:3.0:1788972365.767722:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.767729:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972365.767735:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972365.767754:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:3.0:1788972365.767761:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972365.767769:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972365.767773:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972365.767788:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972365.767793:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91ae0. 00000100:00000010:3.0:1788972365.767807:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91600. 02000000:00000010:3.0:1788972365.767827:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5c00. 00000400:00000200:3.0:1788972365.767831:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972365.767834:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972365.767837:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972365.767840:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092224, portal 25 00000400:00000010:3.0:1788972365.767843:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788972365.767845:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179092224, offset 0 00000400:00000200:3.0:1788972365.767849:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000800:00000200:2.0:1788972365.768111:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972365.768115:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:2.0:1788972365.768120:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972365.768125:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972365.768145:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.768151:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972365.768154:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972365.768156:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972365.768159:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972365.768161:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972365.768163:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [508:508:255:4294967295] 192.168.203.124@tcp LPNI seq info [508:508:7:4294967295] 00000400:00000200:3.0:1788972365.768167:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.768172:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972365.768177:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972365.768181:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788972365.768185:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972365.768189:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972365.768192:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972365.768196:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.778791:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.778816:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972365.778967:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:2.0:1788972365.778976:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:2.0:1788972365.778995:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6680 x1875872179092352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972365.779131:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972365.779136:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:2.0:1788972365.779140:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972365.779145:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972365.779162:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:2.0:1788972365.779164:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:2.0:1788972365.779171:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179092224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972365.782236:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.782247:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.782255:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.782264:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0fb80 00000400:00000200:3.0:1788972365.782270:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972365.782278:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a05 [1] + 224 00000400:00000200:3.0:1788972365.782345:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972365.782351:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972365.782370:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6680 x1875872179092352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972381 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972365.782539:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78370. 00000400:00000200:2.0:1788972365.782550:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:2.0:1788972365.782553:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:2.0:1788972365.782574:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6680 x1875872179092352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972381 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972365.782586:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6680 x1875872179092352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972381 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972365.782618:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4c00. 02000000:00000010:2.0:1788972365.782625:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:2.0:1788972365.782630:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6680. 00000400:00000200:2.0:1788972365.784111:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972365.784117:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972365.784125:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972365.784129:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972365.784134:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972365.784137:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0fb00 00000400:00000200:2.0:1788972365.784142:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972365.784148:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a15 [1] + 224 00000400:00000200:2.0:1788972365.784163:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972365.784168:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972365.784180:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179092224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972381 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972365.794150:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972365.794162:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972365.794165:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972365.794180:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179092224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972381 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972365.794190:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179092224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972381 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972365.794218:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5c00. 02000000:00000010:3.0:1788972365.794224:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972365.794228:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000020:00000010:1.0:1788972366.158571:0:116575:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ad0. 00000020:00000010:1.0:1788972366.158658:0:116575:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ad0. 00000100:00000010:2.0:1788972366.472549:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91ae0. 00000100:00000010:3.0:1788972366.473683:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91600. 00000020:00000010:2.0:1788972367.204011:0:116579:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972367.204102:0:116579:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:3.0:1788972368.254096:0:116583:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972368.254181:0:116583:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:2.0:1788972369.301146:0:116587:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972369.301224:0:116587:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:1.0:1788972370.330018:0:116591:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308d00. 00000020:00000010:1.0:1788972370.330117:0:116591:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308d00. 00000100:00000010:3.0:1788972370.887481:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972370.887514:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972370.887568:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788972370.887575:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 02000000:00000010:3.0:1788972370.887622:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4200. 00000400:00000200:3.0:1788972370.887653:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972370.887658:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972370.887667:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972370.887677:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092608, portal 10 00000400:00000010:3.0:1788972370.887686:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081b28. 00000100:00000200:3.0:1788972370.887689:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179092608, offset 0 00000400:00000200:3.0:1788972370.887700:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972370.887711:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972370.887720:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972370.887727:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972370.887731:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972370.887736:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972370.887739:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972370.887744:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [509:509:256:4294967295] 192.168.203.124@tcp LPNI seq info [509:509:8:4294967295] 00000400:00000200:3.0:1788972370.887748:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972370.887758:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972370.887765:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972370.887770:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:3.0:1788972370.887779:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 02000000:00000010:2.0:1788972370.887964:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548600. 00000400:00000200:2.0:1788972370.887971:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972370.887983:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3000. 00000400:00000010:2.0:1788972370.887989:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972370.887992:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092480, portal 25 00000400:00000010:2.0:1788972370.887996:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8. 00000100:00000200:2.0:1788972370.887998:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179092480, offset 0 00000400:00000200:2.0:1788972370.888005:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972370.888008:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.888012:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972370.888024:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972370.888027:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972370.888029:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972370.888032:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972370.888035:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [510:510:255:4294967295] 192.168.203.124@tcp LPNI seq info [510:510:7:4294967295] 00000400:00000200:2.0:1788972370.888039:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.888044:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972370.888049:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972370.888053:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1300. 00000800:00000200:2.0:1788972370.888057:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972370.888062:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972370.888068:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972370.888084:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972370.888090:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7de0. 00000100:00000010:2.0:1788972370.888103:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f73c0. 00000800:00000200:2.0:1788972370.888262:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972370.888266:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1300. 00000400:00000200:2.0:1788972370.888273:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972370.888279:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972370.888292:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8 00000400:00000010:2.0:1788972370.888295:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8. 00000100:00000200:2.0:1788972370.888356:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179092480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972386 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972370.888678:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972370.888682:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972370.888699:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972370.888762:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972370.888766:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:2.0:1788972370.888770:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972370.888774:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972370.888778:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081b28 00000400:00000010:2.0:1788972370.888780:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081b28. 00000100:00000200:2.0:1788972370.888857:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179092608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972386 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972370.891580:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.891592:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972370.891601:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.891607:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.891612:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.891617:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0fc00 00000400:00000200:2.0:1788972370.891622:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.891629:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a35 [1] + 224 00000400:00000200:2.0:1788972370.891645:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972370.891650:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972370.891667:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179092480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972386 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972370.891781:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3000. 00000400:00000200:3.0:1788972370.891857:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:3.0:1788972370.891861:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:3.0:1788972370.891869:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179092480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972386 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972370.891878:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179092480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972386 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972370.891913:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548600. 02000000:00000010:3.0:1788972370.891936:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972370.891941:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000400:00000200:2.0:1788972370.896958:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.896971:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972370.896994:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.897000:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.897006:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.897011:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0fc80 00000400:00000200:2.0:1788972370.897636:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972370.897646:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a25 [1] + 224 00000400:00000200:2.0:1788972370.897675:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972370.897681:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972370.897698:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179092608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972386 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972370.903077:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972370.903094:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972370.903098:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972370.903116:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179092608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972386 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972370.903129:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179092608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972386 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972370.903174:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4200. 02000000:00000010:3.0:1788972370.903181:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788972370.903187:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000020:00000010:2.0:1788972371.430810:0:116595:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972371.430917:0:116595:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000100:00000010:2.0:1788972371.591742:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7de0. 00000100:00000010:3.0:1788972371.592168:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f73c0. 00000020:00000010:0.0:1788972372.485717:0:116600:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972372.485983:0:116600:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000020:00000010:3.0:1788972373.533485:0:116604:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972373.535275:0:116604:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972374.579234:0:116608:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972374.579587:0:116608:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972375.616652:0:116612:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972375.616741:0:116612:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000100:00000010:1.0:1788972376.007466:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972376.007490:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988000. 00000100:00000010:1.0:1788972376.007543:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6a00. 02000000:00000010:1.0:1788972376.007548:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988600. 02000000:00000010:1.0:1788972376.007583:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124da00. 02000000:00000010:0.0:1788972376.007585:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a000. 00000400:00000200:1.0:1788972376.007593:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972376.007598:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972376.007604:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972376.007611:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092864, portal 10 00000400:00000010:1.0:1788972376.007617:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972376.007620:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179092864, offset 0 00000400:00000200:1.0:1788972376.007628:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972376.007637:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972376.007644:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972376.007649:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972376.007655:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972376.007660:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972376.007662:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972376.007667:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000010:0.0:1788972376.007667:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000200:1.0:1788972376.007675:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [511:511:256:4294967295] 192.168.203.124@tcp LPNI seq info [511:511:8:4294967295] 00000400:00000010:0.0:1788972376.007675:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972376.007682:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092736, portal 25 00000400:00000200:1.0:1788972376.007684:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972376.007688:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972376.007692:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179092736, offset 0 00000400:00000200:1.0:1788972376.007700:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000400:00000200:0.0:1788972376.007703:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000800:00000200:1.0:1788972376.007712:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972376.007713:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000010:1.0:1788972376.007720:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988c00. 00000400:00000200:0.0:1788972376.007721:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000800:00000200:1.0:1788972376.007729:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000400:00000200:0.0:1788972376.007733:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972376.007739:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000800:00000200:1.0:1788972376.007741:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000400:00000200:0.0:1788972376.007744:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000800:00000200:1.0:1788972376.007749:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972376.007749:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972376.007756:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [512:512:255:4294967295] 192.168.203.124@tcp LPNI seq info [512:512:7:4294967295] 00000400:00000200:0.0:1788972376.007761:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972376.007768:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972376.007772:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972376.007776:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000. 00000800:00000200:0.0:1788972376.007781:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972376.007786:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972376.007789:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972376.007837:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972376.007837:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972376.007846:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9300. 00000100:00000010:1.0:1788972376.007857:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9f60. 00000800:00000200:2.0:1788972376.009340:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972376.009448:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988c00. 00000400:00000200:2.0:1788972376.009454:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972376.009461:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972376.009467:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:2.0:1788972376.009470:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:2.0:1788972376.009480:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6a00 x1875872179092864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972376.009557:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972376.009560:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000. 00000400:00000200:2.0:1788972376.009564:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972376.009567:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972376.009570:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990 00000400:00000010:2.0:1788972376.009572:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990. 00000100:00000200:2.0:1788972376.009673:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179092736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972392 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972376.012217:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972376.012226:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972376.012234:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972376.012239:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972376.012244:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972376.012259:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0fd00 00000400:00000200:2.0:1788972376.012264:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972376.012271:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a4d [1] + 224 00000400:00000200:2.0:1788972376.012284:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972376.012289:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972376.012341:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179092736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972392 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972376.012426:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:0.0:1788972376.012431:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:0.0:1788972376.012434:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972376.012439:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179092736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972392 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972376.012447:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179092736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972392 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972376.012473:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a000. 02000000:00000010:0.0:1788972376.012479:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988000. 00000100:00000010:0.0:1788972376.012484:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000400:00000200:3.0:1788972376.013124:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972376.013129:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972376.013135:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972376.013138:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972376.013141:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972376.013145:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0fd80 00000400:00000200:3.0:1788972376.013149:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972376.013153:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a45 [1] + 224 00000400:00000200:3.0:1788972376.013213:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972376.013218:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972376.013226:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6a00 x1875872179092864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972392 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972376.017798:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972376.017812:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770 00000400:00000010:1.0:1788972376.017815:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972376.017828:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6a00 x1875872179092864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972392 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972376.017857:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6a00 x1875872179092864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972392 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972376.017891:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124da00. 02000000:00000010:1.0:1788972376.017898:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988600. 00000100:00000010:1.0:1788972376.017903:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6a00. 00000020:00000010:0.0:1788972376.669711:0:116617:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972376.669784:0:116617:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000100:00000010:2.0:1788972376.711438:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9300. 00000100:00000010:3.0:1788972376.711528:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9f60. 00000020:00000010:0.0:1788972377.711782:0:116621:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972377.711960:0:116621:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000020:00000010:2.0:1788972378.736754:0:116625:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972378.736829:0:116625:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:3.0:1788972379.764825:0:116629:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972379.766553:0:116629:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972380.798654:0:116633:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca470. 00000020:00000010:3.0:1788972380.798745:0:116633:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca470. 00000100:00000010:1.0:1788972381.127449:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6a00. 02000000:00000010:1.0:1788972381.127471:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988600. 00000100:00000010:1.0:1788972381.127528:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972381.127534:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988500. 02000000:00000010:1.0:1788972381.127565:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c000. 00000400:00000200:1.0:1788972381.127573:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972381.127577:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972381.127583:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972381.127590:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179093120, portal 10 00000400:00000010:1.0:1788972381.127596:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588220. 00000100:00000200:1.0:1788972381.127600:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179093120, offset 0 00000400:00000200:1.0:1788972381.127609:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972381.127616:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972381.127623:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972381.127628:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972381.127631:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972381.127635:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972381.127638:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972381.127642:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [513:513:256:4294967295] 192.168.203.124@tcp LPNI seq info [513:513:8:4294967295] 00000400:00000200:1.0:1788972381.127646:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972381.127656:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972381.127662:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972381.127666:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988000. 00000800:00000200:1.0:1788972381.127673:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972381.127707:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972381.127713:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972381.127735:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972381.127740:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d95a0. 00000100:00000010:1.0:1788972381.127754:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9720. 02000000:00000010:1.0:1788972381.127772:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c600. 00000400:00000200:1.0:1788972381.127776:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972381.127778:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972381.127782:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972381.127785:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179092992, portal 25 00000400:00000010:1.0:1788972381.127788:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588cc0. 00000100:00000200:1.0:1788972381.127799:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179092992, offset 0 00000400:00000200:1.0:1788972381.127804:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972381.127808:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972381.127811:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972381.127814:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972381.127816:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972381.127818:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972381.127820:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972381.127823:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [514:514:255:4294967295] 192.168.203.124@tcp LPNI seq info [514:514:7:4294967295] 00000400:00000200:1.0:1788972381.127834:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972381.127840:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972381.127845:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972381.127848:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988c00. 00000800:00000200:1.0:1788972381.127852:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972381.127856:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972381.127858:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972381.127862:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972381.127979:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972381.127985:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988000. 00000400:00000200:2.0:1788972381.127992:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972381.127997:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972381.128002:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588220 00000400:00000010:2.0:1788972381.128004:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588220. 00000100:00000200:2.0:1788972381.128013:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179093120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972381.128089:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972381.128093:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988c00. 00000400:00000200:2.0:1788972381.128096:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972381.128099:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972381.128101:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588cc0 00000400:00000010:2.0:1788972381.128103:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588cc0. 00000100:00000200:2.0:1788972381.128108:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6a00 x1875872179092992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972397 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972381.129544:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972381.129552:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972381.129559:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972381.129563:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972381.129568:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972381.129572:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e0fe80 00000400:00000200:3.0:1788972381.129577:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972381.129582:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a55 [1] + 224 00000400:00000200:3.0:1788972381.129593:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972381.129597:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972381.129607:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179093120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972397 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972381.130709:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972381.130719:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770 00000400:00000010:1.0:1788972381.130721:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972381.130745:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179093120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972397 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972381.130756:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179093120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972397 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972381.130785:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c000. 02000000:00000010:1.0:1788972381.130792:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988500. 00000100:00000010:1.0:1788972381.130797:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000400:00000200:2.0:1788972381.131051:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972381.131057:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972381.131064:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972381.131067:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972381.131072:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972381.131082:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0fe00 00000400:00000200:2.0:1788972381.131086:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972381.131092:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a5d [1] + 224 00000400:00000200:2.0:1788972381.131105:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972381.131109:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972381.131117:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6a00 x1875872179092992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972397 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972381.131188:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972381.131192:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972381.131194:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972381.131199:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6a00 x1875872179092992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972397 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972381.131206:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6a00 x1875872179092992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972397 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972381.131218:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c600. 02000000:00000010:1.0:1788972381.131221:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988600. 00000100:00000010:1.0:1788972381.131224:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6a00. 00000100:00000010:3.0:1788972381.831430:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9720. 00000020:00000010:0.0:1788972381.831682:0:116637:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972381.831760:0:116637:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000100:00000010:2.0:1788972381.835246:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d95a0. 00000020:00000010:2.0:1788972382.866708:0:116641:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972382.866856:0:116641:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:1.0:1788972383.906554:0:116646:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308d00. 00000020:00000010:1.0:1788972383.906643:0:116646:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308d00. 00000020:00000010:2.0:1788972384.948840:0:116650:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972384.948942:0:116650:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000008:00000010:1.0:1788972385.223487:0:115980:0:(osc_request.c:849:osc_shrink_grant_to_target()) kmalloced '(body)': 208 at ffff98bb53988600. 00000100:00000010:1.0:1788972385.223503:0:115980:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6a00. 02000000:00000010:1.0:1788972385.223518:0:115980:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff98bb5124c600. 00000008:00000010:1.0:1788972385.223546:0:115980:0:(osc_request.c:3780:osc_set_info_async()) slab-alloced 'oa': 208 at ffff98bb50fdd680. 02000000:00000010:0.0:1788972385.223627:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff98bb4450a800. 00000400:00000200:0.0:1788972385.223643:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972385.223647:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000010:0.0:1788972385.223655:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972385.223661:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1875872179093248, portal 10 00000400:00000010:0.0:1788972385.223667:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480842a8. 00000100:00000200:0.0:1788972385.223671:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 456 bytes to portal 12, xid 1875872179093248, offset 0 00000400:00000200:0.0:1788972385.223679:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972385.223688:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972385.223695:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972385.223700:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972385.223704:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972385.223708:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972385.223711:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972385.223714:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [515:515:256:4294967295] 192.168.203.124@tcp LPNI seq info [515:515:8:4294967295] 00000400:00000200:0.0:1788972385.223719:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972385.223735:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972385.223742:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 456 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972385.223747:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55c00. 00000800:00000200:0.0:1788972385.223753:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972385.223760:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972385.223765:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55c00 type 1, nob 552 niov 1 nkiov 1 00000400:00000200:0.0:1788972385.223780:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972385.223785:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969180. 00000100:00000010:0.0:1788972385.223808:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969840. 00000008:00000010:1.0:1788972385.223912:0:115980:0:(osc_request.c:877:osc_shrink_grant_to_target()) kfreed 'body': 208 at ffff98bb53988600. 00000800:00000200:3.0:1788972385.224276:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972385.224281:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55c00. 00000400:00000200:3.0:1788972385.224285:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972385.224290:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972385.224296:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480842a8 00000400:00000010:3.0:1788972385.224314:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480842a8. 00000100:00000200:3.0:1788972385.224324:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6a00 x1875872179093248/t0(0) o17->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 456/432 e 0 to 0 dl 1788972401 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972385.232586:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972385.232596:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972385.232606:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972385.232611:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972385.232615:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972385.232619:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 432 into portal 10 MB=0x6aa1886e0ff00 00000400:00000200:2.0:1788972385.232625:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972385.232632:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 432/432 into md 0x1a65 [1] + 224 00000400:00000200:2.0:1788972385.232647:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972385.232651:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972385.232665:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6a00 x1875872179093248/t0(0) o17->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 456/432 e 0 to 0 dl 1788972401 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972385.233420:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:0.0:1788972385.233429:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:0.0:1788972385.233431:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972385.233439:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6a00 x1875872179093248/t0(0) o17->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 456/432 e 0 to 0 dl 1788972401 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972385.233448:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6a00 x1875872179093248/t0(0) o17->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 456/432 e 0 to 0 dl 1788972401 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000010:0.0:1788972385.233498:0:100241:0:(osc_request.c:791:osc_shrink_grant_interpret()) slab-freed '(aa->aa_oa)': 208 at ffff98bb50fdd680. 02000000:00000010:0.0:1788972385.233510:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff98bb4450a800. 02000000:00000010:0.0:1788972385.233776:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff98bb5124c600. 00000100:00000010:0.0:1788972385.233783:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6a00. 00000100:00000010:3.0:1788972385.248431:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969840. 00000100:00000010:2.0:1788972385.248431:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969180. 00000020:00000010:0.0:1788972386.000619:0:116654:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a30. 00000020:00000010:0.0:1788972386.000714:0:116654:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a30. 00000100:00000010:3.0:1788972386.247526:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788972386.247550:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788972386.247786:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972386.247793:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 02000000:00000010:3.0:1788972386.247828:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5c00. 00000400:00000200:3.0:1788972386.247838:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972386.247843:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972386.247850:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972386.247856:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179093504, portal 10 00000400:00000010:3.0:1788972386.247876:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788972386.247879:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179093504, offset 0 00000400:00000200:3.0:1788972386.247888:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972386.247897:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972386.247905:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972386.247910:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972386.247913:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972386.247917:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972386.247919:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972386.247923:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [516:516:256:4294967295] 192.168.203.124@tcp LPNI seq info [516:516:8:4294967295] 00000400:00000200:3.0:1788972386.247927:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972386.247935:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972386.247941:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972386.247945:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00. 00000800:00000200:3.0:1788972386.247951:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972386.247959:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972386.247963:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:2.0:1788972386.247979:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549200. 00000400:00000200:3.0:1788972386.247980:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972386.247987:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91600. 00000400:00000200:2.0:1788972386.247988:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972386.247994:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b39a0. 00000400:00000010:2.0:1788972386.247999:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8. 00000100:00000010:3.0:1788972386.248001:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91ae0. 00000100:00000200:2.0:1788972386.248003:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179093376, portal 25 00000400:00000010:2.0:1788972386.248014:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788972386.248016:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179093376, offset 0 00000400:00000200:2.0:1788972386.248023:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972386.248028:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.248033:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972386.248036:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972386.248038:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972386.248041:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972386.248043:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972386.248046:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [517:517:255:4294967295] 192.168.203.124@tcp LPNI seq info [517:517:7:4294967295] 00000400:00000200:2.0:1788972386.248050:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.248058:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972386.248063:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972386.248067:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1300. 00000800:00000200:2.0:1788972386.248071:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972386.248076:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972386.248079:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972386.248084:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972386.248110:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972386.248117:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00. 00000400:00000200:3.0:1788972386.248121:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972386.248127:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972386.248137:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788972386.248140:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788972386.248147:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179093504/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972386.248661:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972386.248665:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1300. 00000400:00000200:2.0:1788972386.248669:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972386.248674:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972386.248677:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:2.0:1788972386.248679:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788972386.248685:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179093376/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972402 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972386.254712:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.254722:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972386.254730:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.254735:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.254740:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.254757:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e0ff80 00000400:00000200:2.0:1788972386.254762:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.254768:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a7d [1] + 224 00000400:00000200:2.0:1788972386.254782:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972386.254786:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972386.254800:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179093376/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972402 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972386.254918:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b39a0. 00000400:00000200:3.0:1788972386.254923:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8 00000400:00000010:3.0:1788972386.254926:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8. 00000100:00000200:3.0:1788972386.254932:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179093376/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972402 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972386.254942:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179093376/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972402 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972386.254969:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549200. 02000000:00000010:3.0:1788972386.254975:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788972386.254979:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000400:00000200:2.0:1788972386.256935:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.256943:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972386.256951:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.256955:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.256959:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.256963:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10000 00000400:00000200:2.0:1788972386.256967:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972386.256972:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a6d [1] + 224 00000400:00000200:2.0:1788972386.256984:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972386.256988:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972386.256999:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179093504/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972402 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972386.257120:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972386.257126:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972386.257129:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972386.257136:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179093504/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972402 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972386.257144:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179093504/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972402 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972386.257173:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5c00. 02000000:00000010:3.0:1788972386.257179:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972386.257184:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000100:00000010:2.0:1788972386.951422:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91600. 00000100:00000010:3.0:1788972386.951467:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91ae0. 00000020:00000010:2.0:1788972387.038751:0:116658:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166be0. 00000020:00000010:2.0:1788972387.038841:0:116658:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166be0. 00000020:00000010:1.0:1788972387.057229:0:116659:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308d00. 00000020:00000010:1.0:1788972387.059624:0:116659:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308d00. 00000100:00000010:3.0:1788972391.367553:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972391.367585:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 02000000:00000010:2.0:1788972391.367718:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017e00. 00000400:00000200:2.0:1788972391.367736:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972391.367742:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3478. 00000400:00000010:2.0:1788972391.367749:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788972391.367756:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179093632, portal 25 00000400:00000010:2.0:1788972391.367762:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972391.367765:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179093632, offset 0 00000400:00000200:2.0:1788972391.367774:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972391.367796:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.367805:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972391.367809:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972391.367813:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972391.367817:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972391.367820:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972391.367823:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [518:518:256:4294967295] 192.168.203.124@tcp LPNI seq info [518:518:8:4294967295] 00000400:00000200:2.0:1788972391.367839:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.367848:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972391.367855:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972391.367860:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1b00. 00000800:00000200:2.0:1788972391.367867:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972391.367875:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972391.367879:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972391.367900:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972391.367904:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7de0. 00000100:00000010:2.0:1788972391.367917:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f73c0. 00000100:00000010:3.0:1788972391.368169:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788972391.368181:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 00000800:00000200:2.0:1788972391.368259:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972391.368264:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1b00. 00000400:00000200:2.0:1788972391.368269:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972391.368274:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972391.368280:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:2.0:1788972391.368282:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972391.368291:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179093632/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972391.372029:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001800. 00000400:00000200:3.0:1788972391.372072:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972391.372785:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972391.372798:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972391.372809:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179093760, portal 10 00000400:00000010:3.0:1788972391.372820:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788972391.372823:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179093760, offset 0 00000400:00000200:3.0:1788972391.372846:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972391.372861:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972391.372871:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972391.372878:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972391.372882:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972391.372887:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972391.372890:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972391.372895:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [519:519:256:4294967295] 192.168.203.124@tcp LPNI seq info [519:519:8:4294967295] 00000400:00000200:3.0:1788972391.372899:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972391.372910:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972391.372917:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972391.372924:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00. 00000800:00000200:3.0:1788972391.372933:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972391.372943:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972391.372951:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972391.374229:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972391.376967:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972391.376980:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00. 00000400:00000200:2.0:1788972391.376989:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972391.377001:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972391.377010:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:2.0:1788972391.377014:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:2.0:1788972391.377047:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179093760/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972407 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972391.377696:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.377703:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972391.377711:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.377717:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.377723:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.377727:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10080 00000400:00000200:2.0:1788972391.377733:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.377740:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a8d [1] + 224 00000400:00000200:2.0:1788972391.377776:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972391.377780:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972391.377791:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179093632/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972407 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972391.378282:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3478. 00000400:00000200:3.0:1788972391.378288:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:3.0:1788972391.378291:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:3.0:1788972391.378336:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179093632/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972407 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972391.378346:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179093632/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972407 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972391.378378:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017e00. 02000000:00000010:3.0:1788972391.378396:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972391.378402:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000400:00000200:2.0:1788972391.393255:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.393269:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972391.393280:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.393286:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.393293:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.393297:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10100 00000400:00000200:2.0:1788972391.393341:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972391.393349:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1a9d [1] + 224 00000400:00000200:2.0:1788972391.393371:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972391.393377:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972391.393395:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179093760/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972407 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972391.394470:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972391.394482:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972391.394485:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972391.394499:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179093760/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972407 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972391.394512:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179093760/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972407 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972391.394555:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001800. 02000000:00000010:3.0:1788972391.394562:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788972391.394568:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000100:00000010:2.0:1788972392.071879:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7de0. 00000100:00000010:3.0:1788972392.072140:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f73c0. 00000100:00000010:1.0:1788972396.488008:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7b80. 02000000:00000010:1.0:1788972396.488028:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988600. 02000000:00000010:0.0:1788972396.488106:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350b800. 00000400:00000200:0.0:1788972396.488115:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972396.488119:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000010:0.0:1788972396.488126:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972396.488133:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179093888, portal 25 00000400:00000010:0.0:1788972396.488138:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084ee0. 00000100:00000200:0.0:1788972396.488141:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179093888, offset 0 00000400:00000200:0.0:1788972396.488150:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972396.488159:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972396.488166:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972396.488172:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972396.488175:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972396.488179:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972396.488182:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972396.488186:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [520:520:256:4294967295] 192.168.203.124@tcp LPNI seq info [520:520:8:4294967295] 00000400:00000200:0.0:1788972396.488190:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972396.488199:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972396.488205:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972396.488209:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55c00. 00000800:00000200:0.0:1788972396.488215:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972396.488232:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972396.488237:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972396.488256:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972396.488274:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449695a0. 00000100:00000010:0.0:1788972396.488373:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449692a0. 00000800:00000200:2.0:1788972396.488559:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972396.488564:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55c00. 00000400:00000200:2.0:1788972396.488574:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972396.488579:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972396.488585:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084ee0 00000400:00000010:2.0:1788972396.488588:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084ee0. 00000100:00000200:2.0:1788972396.488597:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7b80 x1875872179093888/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788972396.488843:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4380. 02000000:00000010:1.0:1788972396.488855:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988500. 02000000:00000010:0.0:1788972396.489766:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350ac00. 00000400:00000200:0.0:1788972396.489778:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972396.489783:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000010:0.0:1788972396.489941:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480847f8. 00000100:00000200:0.0:1788972396.489958:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094016, portal 10 00000400:00000010:0.0:1788972396.489966:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084d48. 00000100:00000200:0.0:1788972396.489971:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179094016, offset 0 00000400:00000200:0.0:1788972396.489979:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972396.489988:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972396.489997:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972396.490002:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972396.490006:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972396.490009:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972396.490012:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972396.490016:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [521:521:256:4294967295] 192.168.203.124@tcp LPNI seq info [521:521:8:4294967295] 00000400:00000200:0.0:1788972396.490020:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972396.490028:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972396.490034:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972396.490039:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000. 00000800:00000200:0.0:1788972396.490046:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972396.490054:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972396.490058:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972396.490081:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972396.490229:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972396.490233:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000. 00000400:00000200:2.0:1788972396.490238:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972396.490243:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972396.490248:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084d48 00000400:00000010:2.0:1788972396.490251:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084d48. 00000100:00000200:2.0:1788972396.490259:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4380 x1875872179094016/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972412 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972396.492729:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.492738:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972396.492746:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.492751:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.492756:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.492760:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10200 00000400:00000200:2.0:1788972396.492765:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.492771:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1ab5 [1] + 224 00000400:00000200:2.0:1788972396.492783:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972396.492787:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972396.492811:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4380 x1875872179094016/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972412 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972396.493209:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.493214:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972396.493219:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.493230:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.493245:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.493248:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10180 00000400:00000200:2.0:1788972396.493253:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972396.493257:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1aad [1] + 224 00000400:00000200:2.0:1788972396.493265:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972396.493268:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972396.493275:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7b80 x1875872179093888/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972412 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972396.493483:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:0.0:1788972396.493488:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480847f8 00000400:00000010:0.0:1788972396.493491:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480847f8. 00000100:00000200:0.0:1788972396.493498:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4380 x1875872179094016/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972412 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972396.493508:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4380 x1875872179094016/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972412 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972396.493545:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:0.0:1788972396.493548:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:0.0:1788972396.493549:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972396.493554:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7b80 x1875872179093888/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972412 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972396.493563:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7b80 x1875872179093888/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972412 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972396.493576:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350ac00. 02000000:00000010:0.0:1788972396.493581:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988500. 00000100:00000010:0.0:1788972396.493586:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4380. 02000000:00000010:0.0:1788972396.493590:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350b800. 02000000:00000010:0.0:1788972396.493592:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988600. 00000100:00000010:0.0:1788972396.493594:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7b80. 00000100:00000010:3.0:1788972397.191416:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449692a0. 00000100:00000010:2.0:1788972397.191634:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449695a0. 00000100:00000010:3.0:1788972401.607623:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788972401.607644:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788972401.607686:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972401.607690:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 02000000:00000010:3.0:1788972401.607721:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001200. 00000400:00000200:3.0:1788972401.607731:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972401.607735:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972401.607741:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972401.607747:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094272, portal 10 00000400:00000010:3.0:1788972401.607763:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788972401.607766:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179094272, offset 0 00000400:00000200:3.0:1788972401.607774:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972401.607783:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972401.607790:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972401.607795:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972401.607798:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972401.607801:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972401.607816:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972401.607820:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [522:522:256:4294967295] 192.168.203.124@tcp LPNI seq info [522:522:8:4294967295] 00000400:00000200:3.0:1788972401.607825:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972401.607834:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972401.607841:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972401.607845:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500. 00000800:00000200:3.0:1788972401.607852:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972401.607859:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972401.607863:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972401.607876:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972401.607881:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91240. 00000100:00000010:3.0:1788972401.607894:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91d80. 02000000:00000010:3.0:1788972401.607913:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001800. 00000400:00000200:3.0:1788972401.607917:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972401.607920:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78420. 00000400:00000010:3.0:1788972401.607923:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788972401.607926:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094144, portal 25 00000400:00000010:3.0:1788972401.607929:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081770. 00000100:00000200:3.0:1788972401.607931:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179094144, offset 0 00000400:00000200:3.0:1788972401.607935:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972401.607938:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972401.607942:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972401.607944:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972401.607946:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972401.607948:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972401.607950:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972401.607953:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [523:523:255:4294967295] 192.168.203.124@tcp LPNI seq info [523:523:7:4294967295] 00000400:00000200:3.0:1788972401.607956:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972401.607961:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972401.607965:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972401.607973:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00. 00000800:00000200:3.0:1788972401.607976:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972401.607980:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972401.607983:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972401.607987:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972401.608781:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972401.608787:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500. 00000400:00000200:2.0:1788972401.608793:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972401.608799:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972401.608804:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:2.0:1788972401.608806:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:2.0:1788972401.608816:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179094272/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972401.608894:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972401.608896:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00. 00000400:00000200:2.0:1788972401.608899:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972401.608903:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972401.608905:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081770 00000400:00000010:2.0:1788972401.608907:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081770. 00000100:00000200:2.0:1788972401.608912:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179094144/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972417 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972401.615185:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.615195:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972401.615203:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.615208:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.615213:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.615217:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10280 00000400:00000200:2.0:1788972401.615234:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.615240:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1acd [1] + 224 00000400:00000200:2.0:1788972401.615254:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972401.615259:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972401.615273:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179094144/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972417 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972401.618421:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78420. 00000400:00000200:3.0:1788972401.618431:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0 00000400:00000010:3.0:1788972401.618433:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788972401.618446:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179094144/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972417 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972401.618456:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179094144/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972417 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972401.618480:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001800. 02000000:00000010:3.0:1788972401.618485:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788972401.618489:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000400:00000200:2.0:1788972401.621572:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.621582:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972401.621592:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.621597:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.621601:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.621605:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10300 00000400:00000200:2.0:1788972401.621611:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972401.621617:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1abd [1] + 224 00000400:00000200:2.0:1788972401.621645:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972401.621649:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972401.621662:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179094272/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972417 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972401.621785:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972401.621791:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972401.621793:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972401.621809:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179094272/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972417 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972401.621817:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179094272/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972417 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972401.621845:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001200. 02000000:00000010:3.0:1788972401.621852:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:3.0:1788972401.621856:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000100:00000010:2.0:1788972402.311566:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91240. 00000100:00000010:3.0:1788972402.314006:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91d80. 00000100:00000010:2.0:1788972406.727483:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972406.727504:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1b00. 00000100:00000010:2.0:1788972406.727536:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972406.727541:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1a00. 02000000:00000010:3.0:1788972406.727667:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001800. 00000400:00000200:3.0:1788972406.727677:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972406.727680:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972406.727686:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972406.727704:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094528, portal 10 00000400:00000010:3.0:1788972406.727711:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788972406.727714:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179094528, offset 0 00000400:00000200:3.0:1788972406.727734:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972406.727742:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972406.727749:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972406.727755:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972406.727758:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972406.727762:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972406.727764:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972406.727768:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [524:524:256:4294967295] 192.168.203.124@tcp LPNI seq info [524:524:8:4294967295] 00000400:00000200:3.0:1788972406.727772:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972406.727781:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972406.727787:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972406.727792:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:3.0:1788972406.727798:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972406.727804:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972406.727821:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972406.727838:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972406.727843:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91d80. 00000100:00000010:3.0:1788972406.727856:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91ae0. 02000000:00000010:3.0:1788972406.727876:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001200. 00000400:00000200:3.0:1788972406.727880:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972406.727883:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78420. 00000400:00000010:3.0:1788972406.727886:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788972406.727888:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094400, portal 25 00000400:00000010:3.0:1788972406.727892:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480817f8. 00000100:00000200:3.0:1788972406.727894:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179094400, offset 0 00000400:00000200:3.0:1788972406.727897:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972406.727901:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972406.727904:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972406.727912:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972406.727914:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972406.727917:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972406.727919:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972406.727922:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [525:525:255:4294967295] 192.168.203.124@tcp LPNI seq info [525:525:7:4294967295] 00000400:00000200:3.0:1788972406.727925:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972406.727931:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972406.727936:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972406.727939:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d700. 00000800:00000200:3.0:1788972406.727942:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972406.727946:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972406.727949:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972406.727953:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972406.728146:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972406.728151:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:2.0:1788972406.728156:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972406.728161:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972406.728166:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0 00000400:00000010:2.0:1788972406.728168:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0. 00000100:00000200:2.0:1788972406.728177:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179094528/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972422 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972406.728258:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972406.728261:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d700. 00000400:00000200:2.0:1788972406.728264:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972406.728267:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972406.728270:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480817f8 00000400:00000010:2.0:1788972406.728272:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480817f8. 00000100:00000200:2.0:1788972406.728276:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179094400/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972422 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972406.735992:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.736002:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972406.736012:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.736016:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.736021:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.736024:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10380 00000400:00000200:2.0:1788972406.736029:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.736035:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1aed [1] + 224 00000400:00000200:2.0:1788972406.736051:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972406.736055:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972406.736074:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179094400/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972422 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972406.736235:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78420. 00000400:00000200:3.0:1788972406.736240:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0 00000400:00000010:3.0:1788972406.736243:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788972406.736250:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179094400/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972422 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972406.736259:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179094400/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972422 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972406.736285:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001200. 02000000:00000010:3.0:1788972406.736291:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1b00. 00000100:00000010:3.0:1788972406.736297:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000400:00000200:2.0:1788972406.739564:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.739574:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972406.739585:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.739591:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.739596:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.739604:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10400 00000400:00000200:2.0:1788972406.739612:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972406.739618:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1add [1] + 224 00000400:00000200:2.0:1788972406.739815:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972406.739837:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972406.739854:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179094528/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972422 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972406.739972:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972406.739979:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972406.739981:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972406.739993:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179094528/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972422 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972406.740006:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179094528/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972422 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972406.740058:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001800. 02000000:00000010:3.0:1788972406.740064:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1a00. 00000100:00000010:3.0:1788972406.740072:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 00000100:00000010:3.0:1788972407.431689:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91ae0. 00000100:00000010:2.0:1788972407.432422:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91d80. 00000100:00000010:2.0:1788972411.848742:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788972411.848764:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1d00. 00000100:00000010:2.0:1788972411.848803:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410a80. 02000000:00000010:2.0:1788972411.848808:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1f00. 02000000:00000010:3.0:1788972411.848903:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001200. 00000400:00000200:3.0:1788972411.848913:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972411.848917:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972411.848924:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972411.848931:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094656, portal 25 00000400:00000010:3.0:1788972411.848936:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788972411.848939:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179094656, offset 0 00000400:00000200:3.0:1788972411.848947:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972411.848955:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972411.848963:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972411.848968:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972411.848972:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972411.848976:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972411.848979:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972411.848983:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [526:526:256:4294967295] 192.168.203.124@tcp LPNI seq info [526:526:8:4294967295] 00000400:00000200:3.0:1788972411.848988:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972411.848997:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972411.849004:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972411.849009:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:3.0:1788972411.849016:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972411.849023:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972411.849028:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972411.849049:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972411.849054:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91ae0. 00000100:00000010:3.0:1788972411.849068:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91b40. 02000000:00000010:3.0:1788972411.849099:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001800. 00000400:00000200:3.0:1788972411.849104:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972411.849107:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78420. 00000400:00000010:3.0:1788972411.849110:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972411.849112:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094784, portal 10 00000400:00000010:3.0:1788972411.849117:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788972411.849119:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179094784, offset 0 00000400:00000200:3.0:1788972411.849123:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972411.849127:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972411.849130:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972411.849133:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972411.849135:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972411.849137:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972411.849139:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972411.849141:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [527:527:255:4294967295] 192.168.203.124@tcp LPNI seq info [527:527:7:4294967295] 00000400:00000200:3.0:1788972411.849144:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972411.849150:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972411.849154:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972411.849157:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00. 00000800:00000200:3.0:1788972411.849161:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972411.849165:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972411.849167:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972411.849173:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972411.849268:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972411.849272:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:2.0:1788972411.849278:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972411.849284:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972411.849288:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0 00000400:00000010:2.0:1788972411.849290:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0. 00000100:00000200:2.0:1788972411.849317:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179094656/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972427 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972411.849395:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972411.849398:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00. 00000400:00000200:2.0:1788972411.849401:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972411.849404:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972411.849407:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081c38 00000400:00000010:2.0:1788972411.849409:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081c38. 00000100:00000200:2.0:1788972411.849414:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410a80 x1875872179094784/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972427 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972411.850931:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.850939:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972411.850947:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.850950:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.850954:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.850958:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10480 00000400:00000200:2.0:1788972411.850963:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.850988:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1afd [1] + 224 00000400:00000200:2.0:1788972411.851000:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972411.851004:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972411.851013:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179094656/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972427 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972411.851709:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972411.851715:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972411.851717:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972411.851726:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179094656/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972427 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972411.851736:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179094656/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972427 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972411.851762:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001200. 02000000:00000010:3.0:1788972411.851779:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1d00. 00000100:00000010:3.0:1788972411.851982:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000400:00000200:2.0:1788972411.852130:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.852135:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972411.852142:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.852145:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.852149:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.852152:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10500 00000400:00000200:2.0:1788972411.852156:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972411.852161:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b0d [1] + 224 00000400:00000200:2.0:1788972411.852171:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972411.852175:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972411.852182:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410a80 x1875872179094784/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972427 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972411.852468:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78420. 00000400:00000200:3.0:1788972411.852472:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972411.852474:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972411.852481:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410a80 x1875872179094784/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972427 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972411.852491:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410a80 x1875872179094784/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972427 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972411.852521:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001800. 02000000:00000010:3.0:1788972411.852526:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1f00. 00000100:00000010:3.0:1788972411.852529:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410a80. 00000100:00000010:3.0:1788972412.551410:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91b40. 00000100:00000010:2.0:1788972412.551499:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91ae0. 00000100:00000010:0.0:1788972416.967686:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788972416.967708:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55900. 00000100:00000010:0.0:1788972416.967748:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788972416.967754:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 02000000:00000010:0.0:1788972416.967786:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350b800. 00000400:00000200:0.0:1788972416.967795:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972416.967799:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000010:0.0:1788972416.967805:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972416.967812:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095040, portal 10 00000400:00000010:0.0:1788972416.967818:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480847f8. 00000100:00000200:0.0:1788972416.967822:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179095040, offset 0 00000400:00000200:0.0:1788972416.967831:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972416.967839:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972416.967846:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972416.967851:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972416.967854:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972416.967858:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972416.967876:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972416.967880:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [528:528:256:4294967295] 192.168.203.124@tcp LPNI seq info [528:528:8:4294967295] 00000400:00000200:0.0:1788972416.967885:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972416.967896:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972416.967902:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972416.967906:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500. 00000800:00000200:0.0:1788972416.967913:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972416.967921:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972416.967925:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972416.967948:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972416.967953:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969480. 00000100:00000010:0.0:1788972416.967967:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb449691e0. 02000000:00000010:0.0:1788972416.967999:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350ac00. 00000800:00000200:2.0:1788972416.968071:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972416.968076:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500. 00000400:00000200:2.0:1788972416.968082:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972416.968087:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972416.968093:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480847f8 00000400:00000010:2.0:1788972416.968096:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480847f8. 00000100:00000200:2.0:1788972416.968111:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179095040/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788972416.968517:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972416.968523:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000010:0.0:1788972416.968528:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972416.968532:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179094912, portal 25 00000400:00000010:0.0:1788972416.968536:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084a18. 00000100:00000200:0.0:1788972416.968538:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179094912, offset 0 00000400:00000200:0.0:1788972416.968544:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972416.968549:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972416.968563:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972416.968566:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972416.968569:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972416.968572:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972416.968574:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972416.968577:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [529:529:256:4294967295] 192.168.203.124@tcp LPNI seq info [529:529:8:4294967295] 00000400:00000200:0.0:1788972416.968581:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972416.968587:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972416.968591:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972416.969134:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55a00. 00000800:00000200:0.0:1788972416.969141:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972416.969147:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972416.969150:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972416.969228:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972416.976018:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972416.976035:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972416.976045:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972416.976065:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972416.976070:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972416.976074:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10600 00000400:00000200:2.0:1788972416.976079:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972416.976085:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b1d [1] + 224 00000400:00000200:2.0:1788972416.976102:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972416.976107:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972416.976122:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179095040/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972432 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972416.976228:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972416.976232:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55a00. 00000400:00000200:2.0:1788972416.976236:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972416.976241:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972416.976244:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084a18 00000400:00000010:2.0:1788972416.976247:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084a18. 00000100:00000200:2.0:1788972416.976254:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179094912/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972432 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972416.976714:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:0.0:1788972416.976719:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:0.0:1788972416.976721:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972416.976728:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179095040/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972432 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972416.976739:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179095040/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972432 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972416.976769:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350b800. 02000000:00000010:0.0:1788972416.976775:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:0.0:1788972416.976780:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000400:00000200:3.0:1788972416.980856:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972416.980866:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972416.980875:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972416.980880:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972416.980885:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972416.980889:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10580 00000400:00000200:3.0:1788972416.980894:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972416.980901:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b25 [1] + 224 00000400:00000200:3.0:1788972416.980915:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972416.980921:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972416.980947:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179094912/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972432 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972416.985734:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:0.0:1788972416.985748:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198 00000400:00000010:0.0:1788972416.985751:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972416.985766:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179094912/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972432 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972416.985778:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179094912/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972432 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972416.985807:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350ac00. 02000000:00000010:0.0:1788972416.985813:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55900. 00000100:00000010:0.0:1788972416.985818:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000100:00000010:3.0:1788972417.671432:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb449691e0. 00000100:00000010:2.0:1788972417.671532:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969480. 00000100:00000010:3.0:1788972422.091806:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972422.091837:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788972422.091882:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788972422.091886:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00. 02000000:00000010:3.0:1788972422.091951:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001800. 00000400:00000200:3.0:1788972422.091962:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972422.091967:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78420. 02000000:00000010:2.0:1788972422.092150:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016400. 00000400:00000200:2.0:1788972422.092157:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972422.092160:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3420. 00000400:00000010:2.0:1788972422.092168:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972422.092178:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095168, portal 25 00000400:00000010:2.0:1788972422.092184:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788972422.092187:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179095168, offset 0 00000400:00000200:2.0:1788972422.092197:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972422.092207:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.092216:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972422.092221:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972422.092226:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972422.092230:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972422.092239:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972422.092243:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [530:530:256:4294967295] 192.168.203.124@tcp LPNI seq info [530:530:8:4294967295] 00000400:00000200:2.0:1788972422.092249:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.092260:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972422.092267:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972422.092272:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1600. 00000800:00000200:2.0:1788972422.092281:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972422.092289:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972422.092295:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972422.092333:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972422.092338:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7420. 00000100:00000010:2.0:1788972422.092359:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0. 00000800:00000200:2.0:1788972422.092501:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972422.092504:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1600. 00000400:00000200:2.0:1788972422.092510:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972422.092516:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972422.092522:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8 00000400:00000010:2.0:1788972422.092525:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788972422.092535:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179095168/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972438 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972422.096344:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972422.096361:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095296, portal 10 00000400:00000010:3.0:1788972422.096375:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972422.096378:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179095296, offset 0 00000400:00000200:3.0:1788972422.096389:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972422.096401:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972422.096409:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972422.096414:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972422.096419:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972422.096423:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972422.096426:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972422.096430:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [531:531:256:4294967295] 192.168.203.124@tcp LPNI seq info [531:531:8:4294967295] 00000400:00000200:3.0:1788972422.096433:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972422.096442:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972422.096450:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972422.096457:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:3.0:1788972422.096465:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972422.096474:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972422.096480:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972422.096508:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972422.100434:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972422.100444:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:2.0:1788972422.100451:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972422.100459:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972422.100465:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:2.0:1788972422.100468:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:2.0:1788972422.100479:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179095296/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972438 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972422.106243:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.106257:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972422.106290:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.106297:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.106341:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.106346:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10680 00000400:00000200:2.0:1788972422.106354:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.106362:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b2d [1] + 224 00000400:00000200:2.0:1788972422.106387:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972422.106393:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972422.106412:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179095168/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972438 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972422.107630:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3420. 00000400:00000200:3.0:1788972422.107642:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:3.0:1788972422.107645:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:3.0:1788972422.107658:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179095168/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972438 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972422.107670:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179095168/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972438 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972422.107722:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016400. 02000000:00000010:3.0:1788972422.107748:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788972422.107755:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000400:00000200:2.0:1788972422.111913:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.111925:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972422.111944:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.111950:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.111956:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.111961:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10700 00000400:00000200:2.0:1788972422.112127:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972422.112145:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b3d [1] + 224 00000400:00000200:2.0:1788972422.112167:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972422.112173:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972422.112195:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179095296/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972438 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972422.112369:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78420. 00000400:00000200:3.0:1788972422.112375:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972422.112379:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972422.112387:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179095296/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972438 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972422.112397:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179095296/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972438 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972422.113878:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001800. 02000000:00000010:3.0:1788972422.113897:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00. 00000100:00000010:3.0:1788972422.113903:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000100:00000010:2.0:1788972422.791469:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7420. 00000100:00000010:3.0:1788972422.793836:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0. 00000100:00000010:2.0:1788972427.209789:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411c00. 02000000:00000010:2.0:1788972427.209824:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1600. 02000000:00000010:3.0:1788972427.209945:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001800. 00000400:00000200:3.0:1788972427.209958:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972427.209963:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78420. 00000400:00000010:3.0:1788972427.209972:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972427.209981:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095424, portal 25 00000400:00000010:3.0:1788972427.209987:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480813b8. 00000100:00000200:3.0:1788972427.209990:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179095424, offset 0 00000400:00000200:3.0:1788972427.210001:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972427.210012:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972427.210022:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972427.210029:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972427.210033:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972427.210038:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972427.210041:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972427.210045:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [532:532:256:4294967295] 192.168.203.124@tcp LPNI seq info [532:532:8:4294967295] 00000400:00000200:3.0:1788972427.210050:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972427.210060:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972427.210067:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972427.210092:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:3.0:1788972427.210100:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972427.210123:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972427.210129:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972427.210157:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972427.210162:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f91f60. 00000100:00000010:3.0:1788972427.210178:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f914e0. 00000100:00000010:2.0:1788972427.210370:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788972427.210378:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1100. 02000000:00000010:3.0:1788972427.211892:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001600. 00000400:00000200:3.0:1788972427.211908:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972427.214041:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972427.214067:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788972427.214077:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095552, portal 10 00000400:00000010:3.0:1788972427.214089:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972427.214092:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179095552, offset 0 00000400:00000200:3.0:1788972427.215029:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972427.215047:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972427.215061:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972427.215068:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972427.215074:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972427.215078:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972427.215082:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972427.215086:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [533:533:255:4294967295] 192.168.203.124@tcp LPNI seq info [533:533:7:4294967295] 00000400:00000200:3.0:1788972427.215091:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972427.215101:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972427.215119:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972427.215262:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788972427.215272:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972427.215283:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972427.215289:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972427.215315:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972427.215492:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972427.215497:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:3.0:1788972427.215507:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972427.215513:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972427.215522:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480813b8 00000400:00000010:3.0:1788972427.215526:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480813b8. 00000100:00000200:3.0:1788972427.215544:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411c00 x1875872179095424/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972427.215666:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972427.215669:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788972427.215672:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972427.215675:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972427.215678:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972427.215680:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972427.215687:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43412d80 x1875872179095552/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972443 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972427.218513:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.218526:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972427.218554:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.218560:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.218566:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.218571:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10780 00000400:00000200:2.0:1788972427.218589:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.218596:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b4d [1] + 224 00000400:00000200:2.0:1788972427.218616:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972427.218622:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972427.218642:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411c00 x1875872179095424/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972443 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972427.219497:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78420. 00000400:00000200:3.0:1788972427.219506:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972427.219509:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972427.219520:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411c00 x1875872179095424/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972427.219532:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411c00 x1875872179095424/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972443 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972427.219569:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001800. 02000000:00000010:3.0:1788972427.219591:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1600. 00000100:00000010:3.0:1788972427.219599:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411c00. 00000400:00000200:2.0:1788972427.219703:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.219708:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972427.219715:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.219719:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.219723:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.219727:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10800 00000400:00000200:2.0:1788972427.219732:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972427.219738:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b5d [1] + 224 00000400:00000200:2.0:1788972427.219750:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972427.219754:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972427.219764:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43412d80 x1875872179095552/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972443 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972427.219837:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972427.219842:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110 00000400:00000010:3.0:1788972427.219844:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788972427.219849:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43412d80 x1875872179095552/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972443 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972427.219876:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43412d80 x1875872179095552/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972443 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972427.219898:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001600. 02000000:00000010:3.0:1788972427.219902:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1100. 00000100:00000010:3.0:1788972427.219905:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412d80. 00000100:00000010:3.0:1788972427.911475:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f914e0. 00000100:00000010:2.0:1788972427.911655:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f91f60. 00000100:00000010:0.0:1788972432.328572:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788972432.328599:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55900. 00000100:00000010:0.0:1788972432.328643:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788972432.328648:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 02000000:00000010:0.0:1788972432.328688:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a000. 00000400:00000200:0.0:1788972432.328717:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972432.328722:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000010:0.0:1788972432.328729:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972432.328737:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095808, portal 10 00000400:00000010:0.0:1788972432.328746:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788972432.328750:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179095808, offset 0 00000400:00000200:0.0:1788972432.328761:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972432.328771:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972432.328780:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972432.328787:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972432.328791:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972432.328796:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972432.328799:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972432.328802:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [534:534:256:4294967295] 192.168.203.124@tcp LPNI seq info [534:534:8:4294967295] 00000400:00000200:0.0:1788972432.328807:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972432.328819:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972432.328841:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972432.328848:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00. 00000800:00000200:0.0:1788972432.328856:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972432.328865:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 02000000:00000010:1.0:1788972432.329178:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124d800. 00000400:00000200:1.0:1788972432.329186:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972432.329189:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972432.329194:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972432.329197:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095680, portal 25 00000400:00000010:1.0:1788972432.329202:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788972432.329204:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179095680, offset 0 00000400:00000200:1.0:1788972432.329209:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972432.329214:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972432.329218:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972432.329221:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972432.329224:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972432.329226:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972432.329228:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972432.329231:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [535:535:255:4294967295] 192.168.203.124@tcp LPNI seq info [535:535:7:4294967295] 00000400:00000200:1.0:1788972432.329235:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972432.329241:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972432.329246:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972432.329252:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988600. 00000800:00000200:1.0:1788972432.329256:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972432.329261:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972432.329265:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972432.329288:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972432.329294:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9c00. 00000100:00000010:1.0:1788972432.329331:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb455d9180. 00000800:00000200:0.0:1788972432.329410:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972432.329419:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972432.329529:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972432.329535:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988600. 00000400:00000200:2.0:1788972432.329554:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972432.329560:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972432.329566:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770 00000400:00000010:2.0:1788972432.329570:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770. 00000100:00000200:2.0:1788972432.329581:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179095680/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972448 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972432.329692:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972432.329696:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00. 00000400:00000200:2.0:1788972432.329699:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972432.329703:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972432.329707:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:2.0:1788972432.329708:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:2.0:1788972432.329714:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179095808/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972448 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972432.334257:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335052:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972432.335070:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335076:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335083:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335088:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10880 00000400:00000200:2.0:1788972432.335094:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335101:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b75 [1] + 224 00000400:00000200:2.0:1788972432.335122:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972432.335127:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972432.335144:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179095680/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972448 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972432.335197:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335202:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972432.335207:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335209:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335212:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335216:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10900 00000400:00000200:2.0:1788972432.335220:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972432.335225:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b6d [1] + 224 00000400:00000200:2.0:1788972432.335239:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972432.335261:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972432.335271:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179095808/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972448 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972432.335508:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972432.335515:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972432.335518:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972432.335526:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179095680/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972448 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972432.335545:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179095680/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972448 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972432.335588:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124d800. 02000000:00000010:1.0:1788972432.335597:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55900. 00000100:00000010:1.0:1788972432.335602:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000400:00000010:0.0:1788972432.335718:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:0.0:1788972432.335724:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198 00000400:00000010:0.0:1788972432.335726:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972432.335733:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179095808/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972448 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972432.335742:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179095808/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972448 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972432.335763:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a000. 02000000:00000010:0.0:1788972432.335777:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:0.0:1788972432.335780:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000100:00000010:2.0:1788972433.031462:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9c00. 00000100:00000010:3.0:1788972433.032105:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb455d9180. 00000100:00000010:0.0:1788972437.448672:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788972437.448693:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 00000100:00000010:0.0:1788972437.448734:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788972437.448739:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 02000000:00000010:0.0:1788972437.448771:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a000. 00000400:00000200:0.0:1788972437.448781:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 02000000:00000010:1.0:1788972437.448784:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124d800. 00000400:00000010:0.0:1788972437.448787:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000200:1.0:1788972437.448793:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972437.448795:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198. 00000400:00000010:1.0:1788972437.448798:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000100:00000200:0.0:1788972437.448803:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096064, portal 10 00000400:00000010:1.0:1788972437.448805:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972437.448810:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179095936, portal 25 00000400:00000010:0.0:1788972437.448811:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480844c8. 00000400:00000010:1.0:1788972437.448816:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588440. 00000100:00000200:0.0:1788972437.448816:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179096064, offset 0 00000100:00000200:1.0:1788972437.448822:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179095936, offset 0 00000400:00000200:0.0:1788972437.448829:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972437.448841:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972437.448841:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972437.448853:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972437.448859:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972437.448863:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972437.448867:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972437.448870:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972437.448873:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [536:536:256:4294967295] 192.168.203.124@tcp LPNI seq info [536:536:8:4294967295] 00000400:00000200:0.0:1788972437.448878:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972437.448885:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972437.448892:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000400:00000200:1.0:1788972437.448895:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972437.448901:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1788972437.448906:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972437.448909:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000800:00000010:0.0:1788972437.448913:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55900. 00000400:00000200:1.0:1788972437.448914:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972437.448919:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000800:00000200:0.0:1788972437.448925:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000400:00000200:1.0:1788972437.448926:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [537:537:255:4294967295] 192.168.203.124@tcp LPNI seq info [537:537:7:4294967295] 00000400:00000200:1.0:1788972437.448933:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:0.0:1788972437.448937:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000400:00000200:1.0:1788972437.448945:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972437.448945:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55900 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:1.0:1788972437.448954:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972437.448960:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988500. 00000800:00000200:1.0:1788972437.448964:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972437.448969:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972437.448972:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972437.448977:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972437.448980:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972437.448985:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969780. 00000100:00000010:0.0:1788972437.449001:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969a20. 00000800:00000200:2.0:1788972437.456586:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972437.456599:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55900. 00000400:00000200:2.0:1788972437.456607:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972437.456617:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972437.456624:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480844c8 00000400:00000010:2.0:1788972437.456627:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480844c8. 00000100:00000200:2.0:1788972437.456638:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179096064/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972437.456732:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972437.456735:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988500. 00000400:00000200:2.0:1788972437.456739:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972437.456743:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972437.456746:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588440 00000400:00000010:2.0:1788972437.456748:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588440. 00000100:00000200:2.0:1788972437.456753:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179095936/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972453 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972437.475560:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.475570:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972437.475581:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.475586:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.475591:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.475595:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10a00 00000400:00000200:2.0:1788972437.475600:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.475606:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b7d [1] + 224 00000400:00000200:2.0:1788972437.475642:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972437.475647:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972437.475662:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179096064/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972453 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972437.476949:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.476956:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972437.476963:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.476966:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.476970:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.476973:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10980 00000400:00000200:2.0:1788972437.476976:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972437.476980:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b85 [1] + 224 00000400:00000200:2.0:1788972437.476990:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972437.476993:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972437.477000:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179095936/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972453 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972437.477114:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:0.0:1788972437.477120:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198 00000400:00000010:0.0:1788972437.477122:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972437.477130:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179096064/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972437.477139:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179096064/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972453 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972437.477184:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a000. 02000000:00000010:0.0:1788972437.477191:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788972437.477195:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000400:00000010:1.0:1788972437.477426:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972437.477431:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972437.477433:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972437.477440:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179095936/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972453 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972437.477462:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179095936/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972453 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972437.477476:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124d800. 02000000:00000010:1.0:1788972437.477481:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:1.0:1788972437.477484:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000100:00000010:2.0:1788972438.151433:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969780. 00000100:00000010:3.0:1788972438.153764:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969a20. 00000100:00000010:3.0:1788972442.569011:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788972442.569032:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788972442.569074:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972442.569079:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00. 02000000:00000010:3.0:1788972442.569114:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001600. 00000400:00000200:3.0:1788972442.569124:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972442.569128:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972442.569134:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788972442.569141:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096320, portal 10 00000400:00000010:3.0:1788972442.569148:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972442.569152:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179096320, offset 0 00000400:00000200:3.0:1788972442.569160:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972442.569168:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972442.569176:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972442.569181:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972442.569185:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972442.569188:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972442.569191:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972442.569195:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [538:538:256:4294967295] 192.168.203.124@tcp LPNI seq info [538:538:8:4294967295] 00000400:00000200:3.0:1788972442.569211:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972442.569220:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972442.569242:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972442.569247:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:3.0:1788972442.569253:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972442.569261:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972442.569265:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972442.569278:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 02000000:00000010:2.0:1788972442.569423:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016800. 00000400:00000200:2.0:1788972442.569429:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972442.569432:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160. 00000400:00000010:2.0:1788972442.569436:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788972442.569439:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096192, portal 25 00000400:00000010:2.0:1788972442.569443:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972442.569444:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179096192, offset 0 00000400:00000200:2.0:1788972442.569449:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972442.569453:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.569457:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972442.569460:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972442.569463:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972442.569475:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972442.569478:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972442.569480:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [539:539:255:4294967295] 192.168.203.124@tcp LPNI seq info [539:539:7:4294967295] 00000400:00000200:2.0:1788972442.569485:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.569497:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972442.569502:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972442.569506:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1000. 00000800:00000200:2.0:1788972442.569510:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972442.569515:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972442.569518:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972442.569522:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972442.569526:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7420. 00000100:00000010:2.0:1788972442.569536:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840. 00000800:00000200:2.0:1788972442.569597:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972442.569601:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:2.0:1788972442.569608:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972442.569613:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972442.569618:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:2.0:1788972442.569620:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:2.0:1788972442.569629:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179096320/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972458 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972442.569704:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972442.569707:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1000. 00000400:00000200:2.0:1788972442.569710:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972442.569714:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972442.569717:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:2.0:1788972442.569719:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972442.569724:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179096192/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972458 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972442.575505:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575515:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972442.575523:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575528:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575533:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575537:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10b00 00000400:00000200:2.0:1788972442.575556:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575562:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b8d [1] + 224 00000400:00000200:2.0:1788972442.575575:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972442.575579:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972442.575593:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179096320/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972458 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972442.575623:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575626:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972442.575630:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575632:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575635:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575637:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10a80 00000400:00000200:2.0:1788972442.575640:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972442.575645:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1b9d [1] + 224 00000400:00000200:2.0:1788972442.575656:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972442.575659:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972442.575664:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179096192/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972458 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972442.575699:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160. 00000400:00000200:2.0:1788972442.575703:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8 00000400:00000010:2.0:1788972442.575705:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788972442.575711:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179096192/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972458 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972442.575719:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179096192/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972458 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972442.575744:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016800. 02000000:00000010:2.0:1788972442.575750:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:2.0:1788972442.575755:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000400:00000010:3.0:1788972442.575874:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972442.575879:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110 00000400:00000010:3.0:1788972442.575881:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788972442.575887:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179096320/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972458 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972442.575895:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179096320/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972458 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972442.575922:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001600. 02000000:00000010:3.0:1788972442.575925:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00. 00000100:00000010:3.0:1788972442.575928:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000100:00000010:3.0:1788972443.271420:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840. 00000100:00000010:2.0:1788972443.271524:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7420. 00000100:00000010:3.0:1788972447.687740:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788972447.687762:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00. 00000100:00000010:3.0:1788972447.687805:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4700. 02000000:00000010:3.0:1788972447.687810:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:3.0:1788972447.687844:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48001600. 00000400:00000200:3.0:1788972447.687853:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972447.687857:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78790. 00000400:00000010:3.0:1788972447.687864:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788972447.687870:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096576, portal 10 00000400:00000010:3.0:1788972447.687877:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972447.687880:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179096576, offset 0 02000000:00000010:2.0:1788972447.687984:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016800. 00000400:00000200:2.0:1788972447.687990:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972447.687993:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3160. 00000400:00000200:3.0:1788972447.688015:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972447.688024:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.688044:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972447.688049:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972447.688053:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972447.688057:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972447.688059:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972447.688063:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [540:540:256:4294967295] 192.168.203.124@tcp LPNI seq info [540:540:8:4294967295] 00000400:00000200:3.0:1788972447.688066:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.688076:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972447.688082:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972447.688087:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:3.0:1788972447.688094:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972447.688101:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972447.688105:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972447.688121:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972447.688126:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f918a0. 00000100:00000010:3.0:1788972447.688135:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43f914e0. 00000800:00000200:3.0:1788972447.688272:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972447.688276:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:3.0:1788972447.688281:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972447.688286:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000010:2.0:1788972447.691844:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788972447.691857:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096448, portal 25 00000400:00000010:2.0:1788972447.691863:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972447.691866:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179096448, offset 0 00000400:00000200:2.0:1788972447.691875:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972447.691884:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972447.691948:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972447.691953:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972447.691956:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972447.691959:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972447.691962:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972447.691966:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [541:541:255:4294967295] 192.168.203.124@tcp LPNI seq info [541:541:7:4294967295] 00000400:00000200:2.0:1788972447.691971:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972447.691980:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972447.691986:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972447.691992:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1000. 00000800:00000200:2.0:1788972447.691999:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972447.692006:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972447.692010:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972447.692017:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.693806:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972447.693814:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972447.693830:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4700 x1875872179096576/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972447.694021:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972447.694026:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1000. 00000400:00000200:3.0:1788972447.694031:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972447.694036:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972447.694041:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:3.0:1788972447.694043:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:3.0:1788972447.694049:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179096448/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972463 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972447.694092:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.694097:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972447.694103:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.694107:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.694112:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.694116:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10c00 00000400:00000200:3.0:1788972447.694121:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.694127:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1bad [1] + 224 00000400:00000200:3.0:1788972447.694139:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972447.694142:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972447.694148:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4700 x1875872179096576/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972463 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972447.694998:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78790. 00000400:00000200:3.0:1788972447.695004:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110 00000400:00000010:3.0:1788972447.695006:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788972447.695013:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4700 x1875872179096576/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972463 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972447.695022:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4700 x1875872179096576/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972463 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972447.695049:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48001600. 02000000:00000010:3.0:1788972447.695055:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:3.0:1788972447.695059:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4700. 00000400:00000200:3.0:1788972447.696145:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.696254:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972447.696262:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.696266:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.696270:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.696274:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10b80 00000400:00000200:3.0:1788972447.696291:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972447.696298:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1bbd [1] + 224 00000400:00000200:3.0:1788972447.696465:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972447.696470:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972447.696479:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179096448/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972463 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972447.700049:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3160. 00000400:00000200:2.0:1788972447.700061:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8 00000400:00000010:2.0:1788972447.700064:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788972447.700077:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179096448/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972463 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972447.700088:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179096448/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972463 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972447.700115:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016800. 02000000:00000010:2.0:1788972447.700136:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00. 00000100:00000010:2.0:1788972447.700141:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000100:00000010:3.0:1788972448.391445:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f914e0. 00000100:00000010:2.0:1788972448.391503:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43f918a0. 00000100:00000010:0.0:1788972452.807737:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788972452.807758:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788972452.807796:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788972452.807800:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:0.0:1788972452.807831:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a400. 00000400:00000200:0.0:1788972452.807840:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972452.807844:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000010:0.0:1788972452.807851:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972452.807858:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096832, portal 10 00000400:00000010:0.0:1788972452.807865:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084e58. 00000100:00000200:0.0:1788972452.807869:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179096832, offset 0 00000400:00000200:0.0:1788972452.807877:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972452.807885:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972452.807892:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972452.807897:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972452.807901:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972452.807905:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972452.807908:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972452.807911:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [542:542:256:4294967295] 192.168.203.124@tcp LPNI seq info [542:542:8:4294967295] 02000000:00000010:1.0:1788972452.808022:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124d800. 00000400:00000200:1.0:1788972452.808028:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972452.808031:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972452.808035:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972452.808038:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096704, portal 25 00000400:00000010:1.0:1788972452.808041:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588908. 00000100:00000200:1.0:1788972452.808043:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179096704, offset 0 00000400:00000200:1.0:1788972452.808048:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972452.808076:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972452.808085:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972452.808104:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972452.808108:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55a00. 00000800:00000200:0.0:1788972452.808115:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972452.808122:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972452.808126:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972452.808146:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972452.808150:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969ba0. 00000100:00000010:0.0:1788972452.808165:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969a20. 00000800:00000200:2.0:1788972452.808258:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972452.808262:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55a00. 00000400:00000200:2.0:1788972452.808266:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972452.808270:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972452.808275:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084e58 00000400:00000010:2.0:1788972452.808277:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084e58. 00000100:00000200:2.0:1788972452.808537:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179096832/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972452.811410:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972452.811418:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972452.811426:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972452.811430:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972452.811434:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972452.811438:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10d00 00000400:00000200:3.0:1788972452.811443:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972452.811449:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1bcd [1] + 224 00000400:00000200:3.0:1788972452.811461:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972452.811465:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:1.0:1788972452.814748:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972452.814762:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972452.814767:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972452.814771:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972452.814775:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972452.814777:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972452.814781:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [543:543:256:4294967295] 192.168.203.124@tcp LPNI seq info [543:543:8:4294967295] 00000400:00000200:1.0:1788972452.814785:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972452.814795:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972452.814801:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972452.814811:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53988c00. 00000800:00000200:1.0:1788972452.814818:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972452.814825:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972452.814830:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53988c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972452.814843:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972452.815044:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972452.815060:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53988c00. 00000400:00000200:2.0:1788972452.815066:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972452.815071:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972452.815076:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588908 00000400:00000010:2.0:1788972452.815079:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588908. 00000100:00000200:2.0:1788972452.815090:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179096704/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972468 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972452.815269:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179096832/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972468 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972452.815414:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:0.0:1788972452.815420:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198 00000400:00000010:0.0:1788972452.815422:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972452.815429:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179096832/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972468 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972452.815438:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179096832/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972468 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972452.815469:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a400. 02000000:00000010:0.0:1788972452.815476:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788972452.815480:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000400:00000200:2.0:1788972452.820886:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972452.820896:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972452.820904:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972452.820909:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972452.820914:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972452.820918:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10c80 00000400:00000200:2.0:1788972452.820922:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972452.820928:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1bd5 [1] + 224 00000400:00000200:2.0:1788972452.820942:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972452.820946:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972452.820959:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179096704/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972468 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972452.821094:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:1.0:1788972452.821105:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788972452.821108:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972452.821114:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179096704/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972468 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972452.821124:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179096704/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972468 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972452.821149:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124d800. 02000000:00000010:1.0:1788972452.821156:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:1.0:1788972452.821160:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000100:00000010:2.0:1788972453.511583:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969ba0. 00000100:00000010:3.0:1788972453.511687:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969a20. 00000100:00000010:1.0:1788972457.927778:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4e00. 02000000:00000010:1.0:1788972457.927809:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988000. 00000100:00000010:1.0:1788972457.927857:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb5180. 02000000:00000010:1.0:1788972457.927864:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53988c00. 02000000:00000010:0.0:1788972457.928009:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb41343600. 00000400:00000200:0.0:1788972457.928023:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972457.928028:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb481637e8. 00000400:00000010:0.0:1788972457.928036:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788972457.928045:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179096960, portal 25 00000400:00000010:0.0:1788972457.928051:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084880. 00000100:00000200:0.0:1788972457.928055:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179096960, offset 0 00000400:00000200:0.0:1788972457.928065:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972457.928076:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972457.928086:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972457.928093:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972457.928098:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972457.928121:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972457.928124:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972457.928129:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [544:544:256:4294967295] 192.168.203.124@tcp LPNI seq info [544:544:8:4294967295] 00000400:00000200:0.0:1788972457.928135:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972457.928159:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972457.928166:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972457.928172:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00. 00000800:00000200:0.0:1788972457.928181:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972457.928190:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972457.928196:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972457.928469:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972457.928475:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969780. 00000100:00000010:0.0:1788972457.928489:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44969480. 02000000:00000010:0.0:1788972457.928515:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb47ecec00. 00000400:00000200:0.0:1788972457.928520:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972457.928523:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e70. 00000400:00000010:0.0:1788972457.928527:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084b28. 00000100:00000200:0.0:1788972457.928530:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179097088, portal 10 00000400:00000010:0.0:1788972457.928536:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084dd0. 00000100:00000200:0.0:1788972457.928538:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179097088, offset 0 00000400:00000200:0.0:1788972457.928543:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972457.928547:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972457.928551:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972457.928554:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972457.928557:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972457.928559:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972457.928561:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972457.928564:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [545:545:255:4294967295] 192.168.203.124@tcp LPNI seq info [545:545:7:4294967295] 00000400:00000200:0.0:1788972457.928568:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972457.928574:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972457.928589:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972457.928593:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500. 00000800:00000200:0.0:1788972457.928597:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972457.928601:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972457.928604:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972457.928610:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972457.928751:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972457.928757:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00. 00000400:00000200:2.0:1788972457.928766:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972457.928772:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972457.928778:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084880 00000400:00000010:2.0:1788972457.928781:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084880. 00000100:00000200:2.0:1788972457.928793:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4e00 x1875872179096960/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972457.928890:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972457.928893:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500. 00000400:00000200:2.0:1788972457.928896:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972457.928899:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972457.928902:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084dd0 00000400:00000010:2.0:1788972457.928904:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084dd0. 00000100:00000200:2.0:1788972457.928909:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb5180 x1875872179097088/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972473 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972457.950642:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950654:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972457.950663:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950669:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950676:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950681:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e10e00 00000400:00000200:2.0:1788972457.950687:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950693:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1be5 [1] + 224 00000400:00000200:2.0:1788972457.950706:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972457.950719:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972457.950739:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb5180 x1875872179097088/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972473 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972457.950800:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950804:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972457.950809:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950812:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950814:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950817:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e10d80 00000400:00000200:2.0:1788972457.950821:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972457.950825:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1bdd [1] + 224 00000400:00000200:2.0:1788972457.950838:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972457.950841:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972457.950847:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4e00 x1875872179096960/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972473 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972457.950921:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e70. 00000400:00000200:1.0:1788972457.950927:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084b28 00000400:00000010:1.0:1788972457.950930:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084b28. 00000100:00000200:1.0:1788972457.950937:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb5180 x1875872179097088/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972473 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972457.950945:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb5180 x1875872179097088/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972473 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972457.950993:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb481637e8. 00000400:00000200:1.0:1788972457.950996:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198 00000400:00000010:1.0:1788972457.950998:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198. 00000100:00000200:1.0:1788972457.951002:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4e00 x1875872179096960/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972473 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972457.951010:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4e00 x1875872179096960/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972473 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972457.951026:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb47ecec00. 02000000:00000010:1.0:1788972457.951034:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988c00. 00000100:00000010:1.0:1788972457.951040:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb5180. 02000000:00000010:1.0:1788972457.951052:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb41343600. 02000000:00000010:1.0:1788972457.951055:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53988000. 00000100:00000010:1.0:1788972457.951057:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4e00. 00000001:02000400:3.0:1788972458.113967:0:116920:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_310: @@@@@@ FAIL: Failed to cleanup LNet 00000100:00000010:2.0:1788972458.634951:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969780. 00000100:00000010:3.0:1788972458.635098:0:116100:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44969480. Debug log: 3263 lines, 3263 kept, 0 dropped, 0 bad.