00000100:00000010:2.0F:1788971531.207741:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380. 02000000:00000010:2.0:1788971531.207778:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d000. 00000100:00000010:2.0:1788971531.207815:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413800. 02000000:00000010:2.0:1788971531.207820:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700. 02000000:00000010:3.0F:1788971531.207960:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548600. 00000400:00000200:3.0:1788971531.207978:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971531.207982:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78e70. 00000400:00000010:3.0:1788971531.207990:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788971531.207998:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179050496, portal 25 00000400:00000010:3.0:1788971531.208004:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081330. 00000100:00000200:3.0:1788971531.208006:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179050496, offset 0 00000400:00000200:3.0:1788971531.208015:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971531.208023:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.208035:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971531.208040:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971531.208044:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971531.208047:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971531.208050:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971531.208053:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [181:181:256:4294967295] 192.168.203.124@tcp LPNI seq info [181:181:8:4294967295] 00000400:00000200:3.0:1788971531.208057:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.208065:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971531.208071:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971531.208076:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:3.0:1788971531.208082:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971531.208089:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971531.208095:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971531.208116:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971531.208121:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859af60. 00000100:00000010:3.0:1788971531.208131:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a120. 02000000:00000010:3.0:1788971531.208152:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549400. 00000400:00000200:3.0:1788971531.208156:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971531.208159:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e787e8. 00000400:00000010:3.0:1788971531.208162:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788971531.208165:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179050624, portal 10 00000400:00000010:3.0:1788971531.208169:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110. 00000100:00000200:3.0:1788971531.208172:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179050624, offset 0 00000400:00000200:3.0:1788971531.208182:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971531.208186:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.208189:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971531.208191:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971531.208194:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971531.208196:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971531.208198:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971531.208201:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [182:182:255:4294967295] 192.168.203.124@tcp LPNI seq info [182:182:7:4294967295] 00000400:00000200:3.0:1788971531.208205:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.208210:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971531.208214:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971531.208218:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d700. 00000800:00000200:3.0:1788971531.208221:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971531.208225:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971531.208228:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971531.208233:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971531.208399:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971531.208403:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:2.0:1788971531.208409:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971531.208415:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971531.208421:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081330 00000400:00000010:2.0:1788971531.208423:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081330. 00000100:00000200:2.0:1788971531.208434:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410380 x1875872179050496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971547 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971531.208699:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971531.208703:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d700. 00000400:00000200:2.0:1788971531.208706:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971531.208711:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971531.208714:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110 00000400:00000010:2.0:1788971531.208715:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110. 00000100:00000200:2.0:1788971531.208721:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413800 x1875872179050624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971547 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971531.212121:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212130:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971531.212139:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212143:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212148:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212152:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05880 00000400:00000200:3.0:1788971531.212157:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212163:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xab5 [1] + 224 00000400:00000200:3.0:1788971531.212174:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971531.212179:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971531.212192:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413800 x1875872179050624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971547 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971531.212429:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212433:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971531.212450:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212452:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212455:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212457:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05800 00000400:00000200:3.0:1788971531.212461:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971531.212464:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xaa5 [1] + 224 00000400:00000200:3.0:1788971531.212485:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971531.212488:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971531.212495:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410380 x1875872179050496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971547 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971531.212777:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e787e8. 00000400:00000200:2.0:1788971531.212783:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081c38 00000400:00000010:2.0:1788971531.212785:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081c38. 00000100:00000200:2.0:1788971531.212793:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413800 x1875872179050624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971547 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971531.212802:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413800 x1875872179050624/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971547 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971531.212829:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78e70. 00000400:00000200:2.0:1788971531.212832:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:2.0:1788971531.212833:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:2.0:1788971531.212837:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410380 x1875872179050496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971547 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971531.212856:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410380 x1875872179050496/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971547 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971531.212870:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549400. 02000000:00000010:2.0:1788971531.212875:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700. 00000100:00000010:2.0:1788971531.212880:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413800. 02000000:00000010:2.0:1788971531.212884:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548600. 02000000:00000010:2.0:1788971531.212886:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d000. 00000100:00000010:2.0:1788971531.212888:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380. 00000100:00000010:2.0:1788971531.912040:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859af60. 00000100:00000010:3.0:1788971531.914814:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a120. 00000100:00000010:3.0:1788971536.327650:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480. 02000000:00000010:3.0:1788971536.327684:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788971536.327730:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6300. 02000000:00000010:3.0:1788971536.327737:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200. 02000000:00000010:3.0:1788971536.327782:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549000. 00000400:00000200:3.0:1788971536.327794:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971536.327799:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e780b0. 00000400:00000010:3.0:1788971536.327807:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788971536.327816:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179050880, portal 10 00000400:00000010:3.0:1788971536.327826:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971536.327829:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179050880, offset 0 00000400:00000200:3.0:1788971536.327840:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971536.327850:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.327860:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971536.327866:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971536.327871:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971536.327876:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971536.327879:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971536.327884:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [183:183:256:4294967295] 192.168.203.124@tcp LPNI seq info [183:183:8:4294967295] 00000400:00000200:3.0:1788971536.327888:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 02000000:00000010:2.0:1788971536.327997:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:2.0:1788971536.328004:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971536.328007:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3268. 00000400:00000010:2.0:1788971536.328011:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0e58. 00000100:00000200:2.0:1788971536.328015:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179050752, portal 25 00000400:00000010:2.0:1788971536.328019:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788971536.328021:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179050752, offset 0 00000400:00000200:2.0:1788971536.328026:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971536.328063:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971536.328070:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971536.328075:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:3.0:1788971536.328094:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971536.328103:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971536.328108:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971536.328128:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971536.328134:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859aae0. 00000100:00000010:3.0:1788971536.328146:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a420. 00000800:00000200:3.0:1788971536.328330:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971536.328335:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:3.0:1788971536.328341:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971536.328346:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971536.330490:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.330507:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971536.330514:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971536.330519:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971536.330523:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971536.330526:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971536.330530:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [184:184:255:4294967295] 192.168.203.124@tcp LPNI seq info [184:184:7:4294967295] 00000400:00000200:2.0:1788971536.330535:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.330548:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971536.330556:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971536.330570:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d400. 00000800:00000200:2.0:1788971536.330579:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971536.330607:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971536.330613:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971536.330623:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.333058:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0 00000400:00000010:3.0:1788971536.333069:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971536.333090:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6300 x1875872179050880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971536.333141:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.333146:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971536.333153:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.333158:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.333164:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.333169:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05980 00000400:00000200:3.0:1788971536.333174:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971536.333182:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xac5 [1] + 224 00000400:00000200:3.0:1788971536.333200:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971536.333205:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971536.333212:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6300 x1875872179050880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971552 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971536.334169:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e780b0. 00000400:00000200:2.0:1788971536.334181:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:2.0:1788971536.334184:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:2.0:1788971536.334198:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6300 x1875872179050880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971536.334208:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6300 x1875872179050880/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971552 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971536.334246:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549000. 02000000:00000010:2.0:1788971536.334254:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200. 00000100:00000010:2.0:1788971536.334259:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6300. 00000800:00000200:3.0:1788971536.335546:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971536.335574:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d400. 00000400:00000200:3.0:1788971536.335582:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971536.335591:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971536.335600:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:3.0:1788971536.335602:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:3.0:1788971536.335613:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7480 x1875872179050752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971536.335827:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.335833:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971536.335842:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.335846:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.337782:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.337792:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05900 00000400:00000200:2.0:1788971536.337800:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971536.337808:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xad5 [1] + 224 00000400:00000200:2.0:1788971536.337845:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971536.337851:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971536.337871:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7480 x1875872179050752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971552 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971536.338070:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3268. 00000400:00000200:3.0:1788971536.338077:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0e58 00000400:00000010:3.0:1788971536.338080:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0e58. 00000100:00000200:3.0:1788971536.338087:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7480 x1875872179050752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971536.338097:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7480 x1875872179050752/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971552 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971536.338134:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:3.0:1788971536.338143:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788971536.338149:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480. 00000100:00000010:3.0:1788971537.032137:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a420. 00000100:00000010:2.0:1788971537.032401:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859aae0. 00000001:02000400:0.0F:1788971538.381702:0:107901:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 280: Don't panic when request_module fails ========================================================== 12:32:17 (1788971537) 00000020:00000010:1.0F:1788971538.602362:0:107924:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971538.602471:0:107924:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971538.731022:0:107930:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971538.731133:0:107930:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:3.0:1788971539.775566:0:107935:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971539.775676:0:107935:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca610. 00000020:00000010:2.0:1788971540.814882:0:107939:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667c0. 00000020:00000010:2.0:1788971540.815501:0:107939:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667c0. 00000100:00000010:2.0:1788971541.447568:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410380. 02000000:00000010:2.0:1788971541.447620:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0df00. 00000100:00000010:2.0:1788971541.447662:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413800. 02000000:00000010:2.0:1788971541.447670:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700. 02000000:00000010:3.0:1788971541.448784:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548e00. 00000400:00000200:3.0:1788971541.448802:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971541.448808:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971541.448817:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971541.448828:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051008, portal 25 00000400:00000010:3.0:1788971541.448836:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081770. 00000100:00000200:3.0:1788971541.448840:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179051008, offset 0 00000400:00000200:3.0:1788971541.448852:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971541.448874:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971541.448885:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971541.448891:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971541.448910:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971541.448915:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971541.448918:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971541.448923:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [185:185:256:4294967295] 192.168.203.124@tcp LPNI seq info [185:185:8:4294967295] 00000400:00000200:3.0:1788971541.448928:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971541.448939:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971541.448946:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971541.448952:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00. 00000800:00000200:3.0:1788971541.448960:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971541.448969:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971541.448976:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971541.448999:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971541.449076:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a120. 00000100:00000010:3.0:1788971541.449089:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a420. 02000000:00000010:3.0:1788971541.449116:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548c00. 00000400:00000200:3.0:1788971541.449122:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971541.449124:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e783c8. 00000400:00000010:3.0:1788971541.449129:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788971541.449132:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051136, portal 10 00000400:00000010:3.0:1788971541.449138:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971541.449141:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179051136, offset 0 00000400:00000200:3.0:1788971541.449146:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971541.449150:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971541.449153:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971541.449156:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971541.449159:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971541.449161:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971541.449163:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971541.449166:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [186:186:255:4294967295] 192.168.203.124@tcp LPNI seq info [186:186:7:4294967295] 00000400:00000200:3.0:1788971541.449170:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971541.449175:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971541.449189:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971541.449192:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788971541.449196:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971541.449201:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971541.449203:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971541.449210:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971541.449267:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971541.449272:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00. 00000400:00000200:3.0:1788971541.449279:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971541.449284:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971541.449290:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081770 00000400:00000010:3.0:1788971541.449292:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081770. 00000100:00000200:3.0:1788971541.449330:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410380 x1875872179051008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971541.449982:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971541.449988:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788971541.449991:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971541.449997:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971541.450086:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788971541.450089:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971541.450099:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413800 x1875872179051136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971541.462835:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.462849:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971541.462863:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.462879:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.462885:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.462890:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05a00 00000400:00000200:2.0:1788971541.462897:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.462904:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xae5 [1] + 224 00000400:00000200:2.0:1788971541.462919:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971541.462925:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971541.462944:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410380 x1875872179051008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971541.462991:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.462995:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971541.462999:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971541.464382:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:3.0:1788971541.464396:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0 00000400:00000010:3.0:1788971541.464399:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971541.464413:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410380 x1875872179051008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971541.464426:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410380 x1875872179051008/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971541.464463:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548e00. 02000000:00000010:3.0:1788971541.464494:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0df00. 00000100:00000010:3.0:1788971541.464501:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410380. 00000400:00000200:2.0:1788971541.468257:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.468270:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.468279:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05a80 00000400:00000200:2.0:1788971541.468285:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971541.468293:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xaf5 [1] + 224 00000400:00000200:2.0:1788971541.468349:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971541.468354:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971541.468374:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413800 x1875872179051136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971541.469284:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e783c8. 00000400:00000200:3.0:1788971541.469295:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0 00000400:00000010:3.0:1788971541.469538:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788971541.469554:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413800 x1875872179051136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971541.469566:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413800 x1875872179051136/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971541.469610:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548c00. 02000000:00000010:3.0:1788971541.469619:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700. 00000100:00000010:3.0:1788971541.469625:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413800. 00000020:00000010:2.0:1788971541.854195:0:107943:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667c0. 00000020:00000010:2.0:1788971541.855030:0:107943:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667c0. 00000100:00000010:2.0:1788971542.151839:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a120. 00000100:00000010:3.0:1788971542.152231:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a420. 00000020:00000010:3.0:1788971542.906983:0:107947:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971542.907093:0:107947:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971543.958752:0:107951:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971543.958871:0:107951:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca610. 00000020:00000010:2.0:1788971545.010702:0:107956:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667c0. 00000020:00000010:2.0:1788971545.010801:0:107956:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667c0. 00000020:00000010:3.0:1788971546.044640:0:107960:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971546.044746:0:107960:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca610. 00000100:00000010:2.0:1788971546.567515:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413100. 02000000:00000010:2.0:1788971546.567548:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d400. 00000100:00000010:2.0:1788971546.567602:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410700. 02000000:00000010:2.0:1788971546.567608:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d000. 02000000:00000010:2.0:1788971546.567656:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3e00. 00000400:00000200:2.0:1788971546.567669:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971546.567675:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971546.567684:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b03b8. 00000100:00000200:2.0:1788971546.567694:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051392, portal 10 00000400:00000010:2.0:1788971546.567703:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788971546.567707:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179051392, offset 0 00000400:00000200:2.0:1788971546.567719:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971546.567729:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.567739:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971546.567746:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971546.567752:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971546.567756:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971546.567760:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971546.567764:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [187:187:256:4294967295] 192.168.203.124@tcp LPNI seq info [187:187:8:4294967295] 00000400:00000200:2.0:1788971546.567769:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.567785:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971546.567793:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971546.567799:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d300. 00000800:00000200:2.0:1788971546.567807:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971546.567816:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971546.567821:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971546.567845:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971546.567853:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7240. 00000100:00000010:2.0:1788971546.567892:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840. 02000000:00000010:2.0:1788971546.567930:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3600. 00000400:00000200:2.0:1788971546.567935:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971546.567938:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3e70. 00000400:00000010:2.0:1788971546.567941:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971546.567944:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051264, portal 25 00000400:00000010:2.0:1788971546.567948:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788971546.567950:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179051264, offset 0 00000400:00000200:2.0:1788971546.567955:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971546.567958:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.567961:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971546.567964:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971546.567967:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971546.567969:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971546.567971:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971546.567974:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [188:188:255:4294967295] 192.168.203.124@tcp LPNI seq info [188:188:7:4294967295] 00000400:00000200:2.0:1788971546.567978:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.567983:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971546.567987:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971546.567992:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700. 00000800:00000200:2.0:1788971546.567995:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971546.567999:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971546.568002:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971546.568006:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971546.568150:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971546.568156:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d300. 00000400:00000200:3.0:1788971546.568166:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971546.568184:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971546.568191:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:3.0:1788971546.568194:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:3.0:1788971546.568205:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410700 x1875872179051392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971546.568334:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971546.568337:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700. 00000400:00000200:3.0:1788971546.568341:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971546.568344:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971546.568354:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:3.0:1788971546.568356:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:3.0:1788971546.568362:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413100 x1875872179051264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971546.572142:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.572156:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971546.572179:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.572185:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.572191:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.572196:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05b00 00000400:00000200:2.0:1788971546.572202:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971546.572230:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xb15 [1] + 224 00000400:00000200:2.0:1788971546.572614:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971546.572622:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971546.572643:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413100 x1875872179051264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971546.572851:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3e70. 00000400:00000200:3.0:1788971546.572859:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:3.0:1788971546.572870:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:3.0:1788971546.572880:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413100 x1875872179051264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971546.572890:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413100 x1875872179051264/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971546.573049:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3600. 02000000:00000010:3.0:1788971546.573059:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d400. 00000100:00000010:3.0:1788971546.573064:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413100. 00000400:00000200:3.0:1788971546.575679:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971546.575690:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971546.575746:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971546.575752:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971546.575758:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971546.575763:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05b80 00000400:00000200:3.0:1788971546.575769:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971546.575777:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xb05 [1] + 224 00000400:00000200:3.0:1788971546.575792:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971546.575797:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971546.575812:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410700 x1875872179051392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971546.576260:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971546.576268:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b03b8 00000400:00000010:2.0:1788971546.576277:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b03b8. 00000100:00000200:2.0:1788971546.576285:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410700 x1875872179051392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971546.576294:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410700 x1875872179051392/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971546.576349:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3e00. 02000000:00000010:2.0:1788971546.576356:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d000. 00000100:00000010:2.0:1788971546.576362:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410700. 00000020:00000010:3.0:1788971547.084214:0:107964:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971547.085121:0:107964:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca610. 00000100:00000010:2.0:1788971547.272247:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7240. 00000100:00000010:3.0:1788971547.272349:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840. 00000020:00000010:3.0:1788971548.142663:0:107968:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca610. 00000020:00000010:3.0:1788971548.142776:0:107968:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca610. 00000020:00000010:2.0:1788971549.185107:0:107972:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667c0. 00000020:00000010:2.0:1788971549.186019:0:107972:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667c0. 00000020:00000010:2.0:1788971550.230513:0:107976:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971550.230814:0:107976:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:0.0:1788971551.283124:0:107981:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971551.283232:0:107981:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000100:00000010:0.0:1788971551.687527:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788971551.687558:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55f00. 00000100:00000010:0.0:1788971551.687609:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788971551.687615:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400. 02000000:00000010:0.0:1788971551.687659:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aba00. 00000400:00000200:0.0:1788971551.687672:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971551.687677:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163108. 00000400:00000010:0.0:1788971551.687686:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788971551.687696:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051648, portal 10 00000400:00000010:0.0:1788971551.687705:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084198. 00000100:00000200:0.0:1788971551.687710:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179051648, offset 0 00000400:00000200:0.0:1788971551.687721:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971551.687732:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971551.687742:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971551.687749:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971551.687754:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971551.687759:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971551.687762:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971551.687766:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [189:189:256:4294967295] 192.168.203.124@tcp LPNI seq info [189:189:8:4294967295] 00000400:00000200:0.0:1788971551.687771:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971551.687782:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971551.687789:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971551.687804:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55600. 00000800:00000200:0.0:1788971551.687813:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971551.687822:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971551.687829:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971551.687857:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971551.687863:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e847e0. 02000000:00000010:1.0:1788971551.688060:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124de00. 00000400:00000200:1.0:1788971551.688067:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971551.688071:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971551.688076:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788971551.688079:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051520, portal 25 00000400:00000010:1.0:1788971551.688093:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445886e8. 00000100:00000200:1.0:1788971551.688096:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179051520, offset 0 00000400:00000200:1.0:1788971551.688101:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971551.688105:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971551.688109:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971551.688112:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971551.688115:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971551.688117:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971551.688120:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971551.688123:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [190:190:255:4294967295] 192.168.203.124@tcp LPNI seq info [190:190:7:4294967295] 00000400:00000200:1.0:1788971551.688126:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971551.688132:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971551.688137:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971551.688140:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71a00. 00000800:00000200:1.0:1788971551.688144:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971551.688148:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971551.688152:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971551.688156:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971551.688609:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84540. 00000800:00000200:2.0:1788971551.690383:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971551.690393:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55600. 00000400:00000200:2.0:1788971551.690403:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971551.690413:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971551.690422:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084198 00000400:00000010:2.0:1788971551.690425:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084198. 00000100:00000200:2.0:1788971551.690439:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179051648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971567 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971551.690865:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971551.690869:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71a00. 00000400:00000200:2.0:1788971551.691298:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971551.691332:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971551.691338:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445886e8 00000400:00000010:2.0:1788971551.691341:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445886e8. 00000100:00000200:2.0:1788971551.691351:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179051520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971567 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971551.694530:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971551.694543:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971551.694553:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971551.694559:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971551.694566:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971551.694570:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05c80 00000400:00000200:3.0:1788971551.694577:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971551.694586:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xb25 [1] + 224 00000400:00000200:3.0:1788971551.694605:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971551.694610:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971551.694629:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179051648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971567 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971551.699664:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971551.699677:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971551.699687:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971551.699693:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971551.699699:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971551.699704:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05c00 00000400:00000200:2.0:1788971551.699711:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971551.699773:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163108. 00000400:00000200:0.0:1788971551.699781:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:0.0:1788971551.699784:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788971551.699814:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179051648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971567 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971551.699827:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179051648/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971567 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971551.699872:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aba00. 02000000:00000010:0.0:1788971551.699880:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400. 00000100:00000010:0.0:1788971551.699885:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000400:00000200:2.0:1788971551.702766:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xb2d [1] + 224 00000400:00000200:2.0:1788971551.702825:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971551.702831:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971551.704693:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179051520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971567 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971551.705077:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971551.705408:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788971551.705412:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788971551.705427:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179051520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971567 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971551.705466:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179051520/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971567 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971551.705503:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124de00. 02000000:00000010:1.0:1788971551.705512:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55f00. 00000100:00000010:1.0:1788971551.705517:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000020:00000010:1.0:1788971552.334914:0:107985:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971552.335527:0:107985:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000100:00000010:3.0:1788971552.393375:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84540. 00000100:00000010:2.0:1788971552.393580:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e847e0. 00000020:00000010:2.0:1788971553.388049:0:107989:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971553.388159:0:107989:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:0.0:1788971554.430916:0:107993:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971554.431013:0:107993:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971555.472462:0:107997:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971555.472576:0:107997:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000020:00000010:1.0:1788971556.518476:0:108001:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971556.520089:0:108001:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000100:00000010:2.0:1788971556.808503:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410700. 02000000:00000010:2.0:1788971556.808537:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d000. 00000100:00000010:2.0:1788971556.808576:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413b80. 02000000:00000010:2.0:1788971556.808581:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0df00. 02000000:00000010:2.0:1788971556.808805:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:2.0:1788971556.808818:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971556.808822:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971556.808831:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b03b8. 00000100:00000200:2.0:1788971556.808841:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051776, portal 25 00000400:00000010:2.0:1788971556.808848:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0198. 00000100:00000200:2.0:1788971556.808869:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179051776, offset 0 00000400:00000200:2.0:1788971556.808881:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971556.808892:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.808903:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971556.808910:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971556.808915:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971556.808920:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971556.808923:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971556.808927:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [191:191:256:4294967295] 192.168.203.124@tcp LPNI seq info [191:191:8:4294967295] 00000400:00000200:2.0:1788971556.808932:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.808942:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971556.808949:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971556.808956:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d400. 00000800:00000200:2.0:1788971556.808964:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971556.808973:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971556.808979:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971556.808995:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971556.809001:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f79c0. 00000100:00000010:2.0:1788971556.809016:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0. 02000000:00000010:2.0:1788971556.809044:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:2.0:1788971556.809049:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971556.809052:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b37e8. 00000400:00000010:2.0:1788971556.809056:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788971556.809059:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179051904, portal 10 00000400:00000010:2.0:1788971556.809065:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788971556.809067:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179051904, offset 0 00000400:00000200:2.0:1788971556.809071:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971556.809074:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.809077:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971556.809080:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971556.809083:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971556.809091:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971556.809094:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971556.809096:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [192:192:255:4294967295] 192.168.203.124@tcp LPNI seq info [192:192:7:4294967295] 00000400:00000200:2.0:1788971556.809100:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.809106:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971556.809110:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971556.809114:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700. 00000800:00000200:2.0:1788971556.809118:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971556.809122:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971556.809124:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971556.809130:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971556.809170:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971556.809174:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d400. 00000400:00000200:2.0:1788971556.809179:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971556.809184:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971556.809191:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0198 00000400:00000010:2.0:1788971556.809193:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0198. 00000100:00000200:2.0:1788971556.809204:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410700 x1875872179051776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971556.809247:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971556.809250:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700. 00000400:00000200:2.0:1788971556.809252:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971556.809255:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971556.809258:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788971556.809259:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788971556.809263:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413b80 x1875872179051904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971556.824520:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824530:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971556.824540:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824559:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824564:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824568:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05d80 00000400:00000200:2.0:1788971556.824573:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824591:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xb45 [1] + 224 00000400:00000200:2.0:1788971556.824604:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971556.824609:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971556.824624:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413b80 x1875872179051904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971572 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971556.824664:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824668:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971556.824672:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824674:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824676:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824679:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05d00 00000400:00000200:2.0:1788971556.824682:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971556.824687:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xb35 [1] + 224 00000400:00000200:2.0:1788971556.824696:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971556.824699:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971556.824705:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410700 x1875872179051776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971572 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971556.824788:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b37e8. 00000400:00000200:3.0:1788971556.824793:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:3.0:1788971556.824796:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:3.0:1788971556.824802:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413b80 x1875872179051904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971556.824812:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413b80 x1875872179051904/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971572 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971556.824840:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:3.0:1788971556.824843:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b03b8 00000400:00000010:3.0:1788971556.824844:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b03b8. 00000100:00000200:3.0:1788971556.824862:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410700 x1875872179051776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971556.824871:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410700 x1875872179051776/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971572 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971556.824886:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:3.0:1788971556.824899:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0df00. 00000100:00000010:3.0:1788971556.824904:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413b80. 02000000:00000010:3.0:1788971556.824908:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:3.0:1788971556.824911:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d000. 00000100:00000010:3.0:1788971556.824913:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410700. 00000100:00000010:3.0:1788971557.511411:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0. 00000100:00000010:2.0:1788971557.515859:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f79c0. 00000020:00000010:1.0:1788971557.575275:0:108006:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971557.575761:0:108006:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971558.614979:0:108010:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971558.615212:0:108010:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971559.653538:0:108014:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971559.653652:0:108014:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:0.0:1788971560.702277:0:108018:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971560.702471:0:108018:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000020:00000010:1.0:1788971561.740845:0:108022:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971561.741839:0:108022:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000100:00000010:3.0:1788971561.927817:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480. 02000000:00000010:3.0:1788971561.927850:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788971561.927897:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4000. 02000000:00000010:3.0:1788971561.927903:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 02000000:00000010:2.0:1788971561.927982:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2c00. 00000400:00000200:2.0:1788971561.927997:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971561.928002:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788971561.928010:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788971561.928019:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052032, portal 25 00000400:00000010:2.0:1788971561.928025:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0198. 00000100:00000200:2.0:1788971561.928028:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179052032, offset 0 00000400:00000200:2.0:1788971561.928039:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971561.928067:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.928076:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971561.928082:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971561.928088:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971561.928092:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971561.928095:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971561.928098:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [193:193:256:4294967295] 192.168.203.124@tcp LPNI seq info [193:193:8:4294967295] 00000400:00000200:2.0:1788971561.928103:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.928115:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971561.928121:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971561.928127:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700. 00000800:00000200:2.0:1788971561.928135:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971561.928145:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971561.928149:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971561.928211:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971561.928217:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f79c0. 00000100:00000010:2.0:1788971561.928230:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7c00. 02000000:00000010:2.0:1788971561.928254:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3c00. 00000400:00000200:2.0:1788971561.928259:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971561.928262:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971561.928267:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0cc0. 00000100:00000200:2.0:1788971561.928270:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052160, portal 10 00000400:00000010:2.0:1788971561.928279:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b03b8. 00000100:00000200:2.0:1788971561.928281:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179052160, offset 0 00000400:00000200:2.0:1788971561.928286:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971561.928290:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.928293:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971561.928296:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971561.928340:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971561.928344:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971561.928346:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971561.928356:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [194:194:255:4294967295] 192.168.203.124@tcp LPNI seq info [194:194:7:4294967295] 00000400:00000200:2.0:1788971561.928360:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.928366:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971561.928372:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971561.928375:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d400. 00000800:00000200:2.0:1788971561.928379:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971561.928383:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971561.928386:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971561.928392:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971561.934949:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971561.934964:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700. 00000400:00000200:3.0:1788971561.934975:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971561.934986:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971561.934995:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0198 00000400:00000010:3.0:1788971561.934998:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0198. 00000100:00000200:3.0:1788971561.935012:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7480 x1875872179052032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971561.935121:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971561.935125:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d400. 00000400:00000200:3.0:1788971561.935129:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971561.935133:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971561.935136:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b03b8 00000400:00000010:3.0:1788971561.935138:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b03b8. 00000100:00000200:3.0:1788971561.935144:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4000 x1875872179052160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971561.938148:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938160:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971561.938170:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938175:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938181:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938186:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05e00 00000400:00000200:2.0:1788971561.938192:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938199:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xb55 [1] + 224 00000400:00000200:2.0:1788971561.938213:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971561.938218:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971561.938235:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7480 x1875872179052032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971561.938274:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938277:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971561.938280:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938281:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938283:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938285:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05e80 00000400:00000200:2.0:1788971561.938288:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971561.938291:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xb65 [1] + 224 00000400:00000200:2.0:1788971561.938347:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971561.938349:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971561.938355:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4000 x1875872179052160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971561.938655:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:3.0:1788971561.938661:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0cc0 00000400:00000010:3.0:1788971561.938664:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0cc0. 00000100:00000200:3.0:1788971561.938672:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4000 x1875872179052160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971561.938696:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4000 x1875872179052160/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971577 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971561.938742:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:3.0:1788971561.938746:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:3.0:1788971561.938748:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:3.0:1788971561.938753:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7480 x1875872179052032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971561.938761:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7480 x1875872179052032/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971577 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971561.938779:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3c00. 02000000:00000010:3.0:1788971561.938788:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:3.0:1788971561.938794:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4000. 02000000:00000010:3.0:1788971561.938799:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2c00. 02000000:00000010:3.0:1788971561.938802:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788971561.938804:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480. 00000100:00000010:2.0:1788971562.631465:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f79c0. 00000100:00000010:3.0:1788971562.631504:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7c00. 00000020:00000010:3.0:1788971562.763062:0:108026:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971562.763146:0:108026:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:1.0:1788971563.797151:0:108030:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971563.797236:0:108030:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971564.834536:0:108035:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971564.834625:0:108035:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:0.0:1788971565.857414:0:108039:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971565.857487:0:108039:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000020:00000010:1.0:1788971566.890375:0:108043:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971566.890462:0:108043:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000100:00000010:2.0:1788971567.047728:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411c00. 02000000:00000010:2.0:1788971567.047957:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d300. 02000000:00000010:3.0:1788971567.048100:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548800. 00000400:00000200:3.0:1788971567.048110:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971567.048115:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e783c8. 00000400:00000010:3.0:1788971567.048122:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788971567.048128:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052288, portal 25 00000400:00000010:3.0:1788971567.048134:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971567.048137:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179052288, offset 0 00000400:00000200:3.0:1788971567.048145:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971567.048153:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971567.048162:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971567.048167:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971567.048185:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971567.048199:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971567.048202:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971567.048206:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [195:195:256:4294967295] 192.168.203.124@tcp LPNI seq info [195:195:8:4294967295] 00000400:00000200:3.0:1788971567.048211:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971567.048220:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971567.048227:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971567.048232:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788971567.048238:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971567.048246:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971567.048250:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971567.048276:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971567.048282:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859ac00. 00000100:00000010:3.0:1788971567.048294:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a120. 00000100:00000010:2.0:1788971567.048410:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788971567.048418:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d400. 02000000:00000010:3.0:1788971567.048677:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548200. 00000400:00000200:3.0:1788971567.048691:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971567.048694:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971567.048698:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971567.048701:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052416, portal 10 00000400:00000010:3.0:1788971567.048707:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081770. 00000100:00000200:3.0:1788971567.048709:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179052416, offset 0 00000400:00000200:3.0:1788971567.048714:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971567.048718:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971567.048722:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971567.048725:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971567.048728:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971567.048730:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971567.048733:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971567.048736:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [196:196:255:4294967295] 192.168.203.124@tcp LPNI seq info [196:196:7:4294967295] 00000400:00000200:3.0:1788971567.048740:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971567.048746:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971567.048758:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971567.048762:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dd00. 00000800:00000200:3.0:1788971567.048766:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971567.048771:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971567.048774:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971567.048779:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971567.049002:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971567.049006:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788971567.049012:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971567.049017:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971567.049023:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0 00000400:00000010:3.0:1788971567.049025:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971567.049034:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411c00 x1875872179052288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971567.049740:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971567.049744:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dd00. 00000400:00000200:3.0:1788971567.049747:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971567.049752:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971567.049756:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081770 00000400:00000010:3.0:1788971567.049758:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081770. 00000100:00000200:3.0:1788971567.049764:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43412d80 x1875872179052416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971567.054515:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.054524:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971567.054533:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.054537:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.054542:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.054546:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e05f00 00000400:00000200:2.0:1788971567.054551:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.054556:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xb75 [1] + 224 00000400:00000200:2.0:1788971567.054570:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971567.054574:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971567.054587:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411c00 x1875872179052288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971583 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971567.056598:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e783c8. 00000400:00000200:3.0:1788971567.056608:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0 00000400:00000010:3.0:1788971567.056611:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788971567.056623:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411c00 x1875872179052288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971583 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971567.056634:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411c00 x1875872179052288/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971583 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971567.056662:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548800. 00000400:00000200:2.0:1788971567.057185:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.057191:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971567.057198:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.057203:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.057207:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.057211:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e05f80 00000400:00000200:2.0:1788971567.057215:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971567.057220:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xb85 [1] + 224 00000400:00000200:2.0:1788971567.057249:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971567.057255:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971567.057265:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43412d80 x1875872179052416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971583 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971567.057775:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d300. 00000100:00000010:3.0:1788971567.057780:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411c00. 00000400:00000010:3.0:1788971567.057791:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:3.0:1788971567.057794:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788971567.057796:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971567.057803:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43412d80 x1875872179052416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971583 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971567.057813:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43412d80 x1875872179052416/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971583 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971567.057832:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548200. 02000000:00000010:3.0:1788971567.057835:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d400. 00000100:00000010:3.0:1788971567.057838:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412d80. 00000100:00000010:3.0:1788971567.751421:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a120. 00000100:00000010:2.0:1788971567.752002:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859ac00. 00000020:00000010:2.0:1788971567.945976:0:108047:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971567.946094:0:108047:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:1.0:1788971568.978594:0:108051:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971568.978773:0:108051:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971570.011033:0:108055:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971570.011118:0:108055:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971571.060423:0:108059:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971571.060520:0:108059:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:0.0:1788971572.106240:0:108064:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971572.106366:0:108064:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000100:00000010:0.0:1788971572.168845:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788971572.168866:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55400. 00000100:00000010:0.0:1788971572.168905:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788971572.168912:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55f00. 02000000:00000010:0.0:1788971572.168946:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa000. 00000400:00000200:0.0:1788971572.168956:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971572.168960:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163108. 00000400:00000010:0.0:1788971572.168967:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788971572.168973:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052672, portal 10 00000400:00000010:0.0:1788971572.168981:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084550. 00000100:00000200:0.0:1788971572.168999:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179052672, offset 0 00000400:00000200:0.0:1788971572.169008:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971572.169017:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971572.169026:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971572.169031:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971572.169035:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971572.169039:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971572.169042:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971572.169045:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [197:197:256:4294967295] 192.168.203.124@tcp LPNI seq info [197:197:8:4294967295] 00000400:00000200:0.0:1788971572.169050:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971572.169060:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971572.169067:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971572.169072:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55600. 00000800:00000200:0.0:1788971572.169078:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971572.169086:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971572.169090:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971572.169112:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971572.169118:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84360. 00000100:00000010:0.0:1788971572.169131:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84120. 02000000:00000010:0.0:1788971572.169162:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aae00. 00000400:00000200:0.0:1788971572.169167:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971572.169170:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163d68. 00000400:00000010:0.0:1788971572.169173:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480843b8. 00000100:00000200:0.0:1788971572.169176:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052544, portal 25 00000400:00000010:0.0:1788971572.169180:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084440. 00000100:00000200:0.0:1788971572.169182:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179052544, offset 0 00000400:00000200:0.0:1788971572.169186:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971572.169190:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971572.169194:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971572.169197:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971572.169199:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971572.169207:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971572.169209:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971572.169212:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [198:198:255:4294967295] 192.168.203.124@tcp LPNI seq info [198:198:7:4294967295] 00000400:00000200:0.0:1788971572.169216:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971572.169223:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971572.169228:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971572.169231:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55000. 00000800:00000200:0.0:1788971572.169235:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971572.169238:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971572.169241:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971572.169245:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971572.169364:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971572.169369:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55600. 00000400:00000200:2.0:1788971572.169375:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971572.169380:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971572.169386:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084550 00000400:00000010:2.0:1788971572.169388:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084550. 00000100:00000200:2.0:1788971572.169397:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179052672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971572.169474:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971572.169477:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55000. 00000400:00000200:2.0:1788971572.169480:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971572.169484:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971572.169486:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084440 00000400:00000010:2.0:1788971572.169488:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084440. 00000100:00000200:2.0:1788971572.169493:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179052544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971572.174337:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.174348:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971572.174357:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.174362:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.174367:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.174372:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06000 00000400:00000200:2.0:1788971572.174377:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.174398:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xb9d [1] + 224 00000400:00000200:2.0:1788971572.174415:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971572.174420:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971572.174439:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179052544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971588 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971572.174544:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163d68. 00000400:00000200:0.0:1788971572.174549:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480843b8 00000400:00000010:0.0:1788971572.174552:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480843b8. 00000100:00000200:0.0:1788971572.174559:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179052544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971588 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971572.174567:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179052544/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971588 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971572.174593:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aae00. 02000000:00000010:0.0:1788971572.174599:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55400. 00000100:00000010:0.0:1788971572.174603:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000400:00000200:2.0:1788971572.181442:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.181452:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971572.181528:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.181534:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.181539:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.181543:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06080 00000400:00000200:2.0:1788971572.181549:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971572.181555:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xb95 [1] + 224 00000400:00000200:2.0:1788971572.181578:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971572.181583:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971572.181596:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179052672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971588 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971572.181752:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163108. 00000400:00000200:0.0:1788971572.181757:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:0.0:1788971572.181760:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788971572.181766:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179052672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971588 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971572.181774:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179052672/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971588 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971572.181805:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa000. 02000000:00000010:0.0:1788971572.181825:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55f00. 00000100:00000010:0.0:1788971572.181829:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000100:00000010:2.0:1788971572.871418:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84360. 00000100:00000010:3.0:1788971572.871422:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84120. 00000020:00000010:3.0:1788971573.137567:0:108068:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971573.137859:0:108068:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:1.0:1788971574.179177:0:108072:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971574.180067:0:108072:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971575.231295:0:108076:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971575.231408:0:108076:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:2.0:1788971576.257747:0:108080:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971576.257840:0:108080:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000100:00000010:1.0:1788971577.294141:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80. 02000000:00000010:1.0:1788971577.294172:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71600. 00000100:00000010:1.0:1788971577.294212:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4700. 02000000:00000010:1.0:1788971577.294217:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71f00. 02000000:00000010:0.0:1788971577.294776:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa000. 00000400:00000200:0.0:1788971577.294788:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971577.294792:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163108. 00000400:00000010:0.0:1788971577.294797:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788971577.294804:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052800, portal 25 00000400:00000010:0.0:1788971577.294809:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480843b8. 00000100:00000200:0.0:1788971577.294813:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179052800, offset 0 00000400:00000200:0.0:1788971577.294821:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971577.294829:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971577.294836:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971577.294842:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971577.294846:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971577.294849:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971577.294852:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971577.294855:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [199:199:256:4294967295] 192.168.203.124@tcp LPNI seq info [199:199:8:4294967295] 00000400:00000200:0.0:1788971577.294859:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971577.294869:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971577.294875:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971577.294880:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55f00. 00000800:00000200:0.0:1788971577.294886:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971577.294895:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971577.294899:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971577.294920:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971577.294925:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84420. 00000100:00000010:0.0:1788971577.294934:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e841e0. 02000000:00000010:0.0:1788971577.294956:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab600. 00000400:00000200:0.0:1788971577.294961:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971577.294963:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163d68. 00000400:00000010:0.0:1788971577.294966:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788971577.294975:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179052928, portal 10 00000400:00000010:0.0:1788971577.294980:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084908. 00000100:00000200:0.0:1788971577.294982:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179052928, offset 0 00000400:00000200:0.0:1788971577.294987:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971577.294991:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971577.294994:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971577.294996:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971577.294999:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971577.295001:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971577.295002:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971577.295005:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [200:200:255:4294967295] 192.168.203.124@tcp LPNI seq info [200:200:7:4294967295] 00000400:00000200:0.0:1788971577.295008:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971577.295014:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971577.295018:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971577.295021:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55400. 00000800:00000200:0.0:1788971577.295025:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971577.295029:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971577.295032:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971577.295038:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971577.295481:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971577.295486:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55f00. 00000400:00000200:3.0:1788971577.295490:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971577.295495:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971577.295500:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480843b8 00000400:00000010:3.0:1788971577.295503:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480843b8. 00000100:00000200:3.0:1788971577.295511:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6d80 x1875872179052800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971577.295576:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971577.295579:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55400. 00000400:00000200:3.0:1788971577.295582:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971577.295585:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971577.295588:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084908 00000400:00000010:3.0:1788971577.295589:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084908. 00000100:00000200:3.0:1788971577.295594:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4700 x1875872179052928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971577.298279:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298289:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971577.298297:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298316:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298321:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298325:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06100 00000400:00000200:2.0:1788971577.298330:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298337:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xba5 [1] + 224 00000400:00000200:2.0:1788971577.298351:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971577.298362:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971577.298376:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6d80 x1875872179052800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971593 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971577.298484:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163108. 00000400:00000200:1.0:1788971577.298490:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:1.0:1788971577.298493:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:1.0:1788971577.298500:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6d80 x1875872179052800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971577.298510:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6d80 x1875872179052800/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971593 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971577.298536:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa000. 02000000:00000010:1.0:1788971577.298542:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71600. 00000100:00000010:1.0:1788971577.298547:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80. 00000400:00000200:2.0:1788971577.298928:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298933:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971577.298938:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298941:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.298944:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.299031:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06180 00000400:00000200:2.0:1788971577.299036:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971577.299041:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xbad [1] + 224 00000400:00000200:2.0:1788971577.299053:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971577.299057:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971577.299065:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4700 x1875872179052928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971593 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971577.299760:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163d68. 00000400:00000200:0.0:1788971577.299768:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:0.0:1788971577.299770:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788971577.299778:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4700 x1875872179052928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971577.299786:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4700 x1875872179052928/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971593 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971577.299815:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab600. 02000000:00000010:0.0:1788971577.299820:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71f00. 00000100:00000010:0.0:1788971577.299824:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4700. 00000020:00000010:3.0:1788971577.311397:0:108084:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971577.311827:0:108084:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000100:00000010:3.0:1788971577.991593:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e841e0. 00000100:00000010:2.0:1788971577.991690:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84420. 00000020:00000010:2.0:1788971578.340274:0:108089:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971578.340386:0:108089:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:0.0:1788971579.409787:0:108093:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971579.409880:0:108093:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000020:00000010:3.0:1788971580.468736:0:108097:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971580.468825:0:108097:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:2.0:1788971581.503539:0:108101:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971581.503634:0:108101:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000100:00000010:1.0:1788971582.407425:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80. 02000000:00000010:1.0:1788971582.407448:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71600. 00000100:00000010:1.0:1788971582.407488:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7100. 02000000:00000010:1.0:1788971582.407494:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71300. 02000000:00000010:0.0:1788971582.407664:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa000. 00000400:00000200:0.0:1788971582.407673:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971582.407678:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163d68. 00000400:00000010:0.0:1788971582.407684:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788971582.407691:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053056, portal 25 00000400:00000010:0.0:1788971582.407696:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084bb0. 00000100:00000200:0.0:1788971582.407700:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179053056, offset 0 00000400:00000200:0.0:1788971582.407709:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971582.407717:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971582.407725:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971582.407730:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971582.407734:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971582.407737:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971582.407740:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971582.407744:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [201:201:256:4294967295] 192.168.203.124@tcp LPNI seq info [201:201:8:4294967295] 00000400:00000200:0.0:1788971582.407749:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971582.407758:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971582.407764:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971582.407769:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55300. 00000800:00000200:0.0:1788971582.407776:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971582.407783:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971582.407788:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971582.407872:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971582.407880:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84ea0. 00000100:00000010:0.0:1788971582.407889:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84840. 02000000:00000010:0.0:1788971582.407919:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aae00. 00000400:00000200:0.0:1788971582.407933:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971582.407936:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971582.407939:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971582.407941:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053184, portal 10 00000400:00000010:0.0:1788971582.407959:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084088. 00000100:00000200:0.0:1788971582.407962:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179053184, offset 0 00000400:00000200:0.0:1788971582.407967:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971582.407971:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971582.407974:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971582.407977:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971582.407979:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971582.407981:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971582.407984:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971582.407986:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [202:202:255:4294967295] 192.168.203.124@tcp LPNI seq info [202:202:7:4294967295] 00000400:00000200:0.0:1788971582.407990:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971582.407996:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971582.408000:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971582.408003:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55c00. 00000800:00000200:0.0:1788971582.408007:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971582.408011:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971582.408014:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971582.408019:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971582.408712:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971582.408718:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55300. 00000400:00000200:2.0:1788971582.408724:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971582.408730:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971582.408736:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084bb0 00000400:00000010:2.0:1788971582.408738:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084bb0. 00000100:00000200:2.0:1788971582.408747:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6d80 x1875872179053056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971582.408813:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971582.408816:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55c00. 00000400:00000200:2.0:1788971582.408826:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971582.408830:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971582.408833:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084088 00000400:00000010:2.0:1788971582.408835:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084088. 00000100:00000200:2.0:1788971582.408841:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7100 x1875872179053184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971582.419880:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.419890:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971582.419899:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.419905:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.419910:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.419913:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06200 00000400:00000200:2.0:1788971582.419919:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.420068:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xbb5 [1] + 224 00000400:00000200:2.0:1788971582.420083:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971582.420088:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971582.420102:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6d80 x1875872179053056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971582.420654:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.420659:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971582.420664:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.420667:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.420670:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.420672:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06280 00000400:00000200:2.0:1788971582.420676:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971582.420680:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xbbd [1] + 224 00000400:00000200:2.0:1788971582.420705:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971582.420709:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971582.420717:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7100 x1875872179053184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971582.420784:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163d68. 00000400:00000200:0.0:1788971582.420790:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:0.0:1788971582.420792:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788971582.420798:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6d80 x1875872179053056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971582.420807:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6d80 x1875872179053056/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971582.420983:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa000. 02000000:00000010:0.0:1788971582.420990:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71600. 00000100:00000010:0.0:1788971582.420995:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80. 00000400:00000010:0.0:1788971582.421003:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971582.421005:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971582.421013:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971582.421018:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7100 x1875872179053184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971582.421026:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7100 x1875872179053184/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971582.421043:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aae00. 02000000:00000010:0.0:1788971582.421046:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71300. 00000100:00000010:0.0:1788971582.421048:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7100. 00000020:00000010:0.0:1788971582.543786:0:108105:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971582.545272:0:108105:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000100:00000010:2.0:1788971583.111697:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84ea0. 00000100:00000010:3.0:1788971583.111815:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84840. 00000020:00000010:2.0:1788971583.596166:0:108109:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971583.596256:0:108109:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:1.0:1788971584.644377:0:108113:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971584.644466:0:108113:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971585.683772:0:108118:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971585.683863:0:108118:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:0.0:1788971586.716902:0:108122:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971586.716991:0:108122:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000100:00000010:0.0:1788971587.528223:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788971587.528245:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788971587.528287:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788971587.528292:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:1.0:1788971587.528414:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124d000. 00000400:00000200:1.0:1788971587.528423:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971587.528438:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971587.528446:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788971587.528453:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053312, portal 25 00000400:00000010:1.0:1788971587.528458:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588c38. 00000100:00000200:1.0:1788971587.528475:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179053312, offset 0 00000400:00000200:1.0:1788971587.528485:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971587.528494:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971587.528507:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971587.528512:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971587.528516:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971587.528521:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971587.528524:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971587.528528:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [203:203:256:4294967295] 192.168.203.124@tcp LPNI seq info [203:203:8:4294967295] 00000400:00000200:1.0:1788971587.528540:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971587.528552:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971587.528558:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971587.528563:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71200. 00000800:00000200:1.0:1788971587.528570:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971587.528578:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971587.528583:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971587.528603:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971587.528608:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f840. 00000100:00000010:1.0:1788971587.528617:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f180. 02000000:00000010:1.0:1788971587.528637:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124ca00. 00000400:00000200:1.0:1788971587.528641:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971587.528644:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971587.528647:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971587.528650:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053440, portal 10 00000400:00000010:1.0:1788971587.528655:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788971587.528657:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179053440, offset 0 00000400:00000200:1.0:1788971587.528661:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971587.528665:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971587.528667:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971587.528670:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971587.528673:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971587.528681:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971587.528683:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971587.528686:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [204:204:255:4294967295] 192.168.203.124@tcp LPNI seq info [204:204:7:4294967295] 00000400:00000200:1.0:1788971587.528689:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971587.528696:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971587.528701:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971587.528704:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71100. 00000800:00000200:1.0:1788971587.528708:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971587.528711:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971587.528714:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971587.528719:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971587.528825:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971587.528829:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71200. 00000400:00000200:2.0:1788971587.528834:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971587.528839:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971587.528844:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588c38 00000400:00000010:2.0:1788971587.528846:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588c38. 00000100:00000200:2.0:1788971587.528854:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179053312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971587.528911:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971587.528913:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71100. 00000400:00000200:2.0:1788971587.528916:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971587.528920:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971587.528922:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445882a8 00000400:00000010:2.0:1788971587.528924:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445882a8. 00000100:00000200:2.0:1788971587.528928:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179053440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971587.537598:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537608:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971587.537619:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537623:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537629:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537633:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06300 00000400:00000200:2.0:1788971587.537639:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537644:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xbc5 [1] + 224 00000400:00000200:2.0:1788971587.537655:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971587.537660:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971587.537675:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179053312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971603 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971587.537812:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537816:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971587.537821:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537823:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537826:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537829:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06380 00000400:00000200:2.0:1788971587.537833:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971587.537837:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xbcd [1] + 224 00000400:00000200:2.0:1788971587.537848:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971587.537852:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971587.537858:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179053440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971603 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971587.537918:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971587.537923:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:1.0:1788971587.537926:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788971587.537932:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179053312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971603 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971587.537942:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179053312/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971603 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971587.537964:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124d000. 02000000:00000010:1.0:1788971587.537971:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:1.0:1788971587.537974:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000400:00000010:1.0:1788971587.537980:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971587.537981:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:1.0:1788971587.537983:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971587.537986:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179053440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971603 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971587.537992:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179053440/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971603 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971587.538006:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124ca00. 02000000:00000010:1.0:1788971587.538016:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:1.0:1788971587.538019:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000020:00000010:3.0:1788971587.756868:0:108126:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971587.756969:0:108126:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000100:00000010:3.0:1788971588.231515:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f180. 00000100:00000010:2.0:1788971588.231556:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f840. 00000020:00000010:3.0:1788971588.782280:0:108130:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971588.782391:0:108130:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:2.0:1788971589.818101:0:108134:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971589.818186:0:108134:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:3.0:1788971590.851169:0:108138:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971590.851267:0:108138:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:1.0:1788971591.887947:0:108142:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971591.888044:0:108142:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000100:00000010:2.0:1788971592.650900:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788971592.650953:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700. 00000100:00000010:2.0:1788971592.650982:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410a80. 02000000:00000010:2.0:1788971592.650999:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d000. 02000000:00000010:2.0:1788971592.651106:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:2.0:1788971592.651116:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971592.651121:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c60. 00000400:00000010:2.0:1788971592.651126:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788971592.651133:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053696, portal 10 00000400:00000010:2.0:1788971592.651139:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971592.651142:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179053696, offset 0 00000400:00000200:2.0:1788971592.651150:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971592.651171:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.651179:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971592.651208:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971592.651213:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971592.651216:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971592.651219:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971592.651223:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [205:205:256:4294967295] 192.168.203.124@tcp LPNI seq info [205:205:8:4294967295] 00000400:00000200:2.0:1788971592.651227:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.651235:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971592.651241:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971592.651245:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0df00. 00000800:00000200:2.0:1788971592.651252:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971592.651258:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971592.651263:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971592.651278:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971592.651282:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7cc0. 00000100:00000010:2.0:1788971592.651293:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7780. 02000000:00000010:3.0:1788971592.651385:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548e00. 00000400:00000200:3.0:1788971592.651392:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971592.651396:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971592.651400:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971592.651404:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053568, portal 25 00000400:00000010:3.0:1788971592.651408:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788971592.651410:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179053568, offset 0 00000400:00000200:3.0:1788971592.651416:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971592.651420:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971592.651426:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000800:00000200:2.0:1788971592.651426:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:3.0:1788971592.651432:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000010:2.0:1788971592.651432:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0df00. 00000400:00000200:3.0:1788971592.651438:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971592.651439:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971592.651443:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971592.651450:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971592.651451:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971592.651456:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [206:206:255:4294967295] 192.168.203.124@tcp LPNI seq info [206:206:7:4294967295] 00000400:00000200:3.0:1788971592.651461:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971592.651467:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971592.651479:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971592.651481:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000800:00000010:3.0:1788971592.651484:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1da00. 00000400:00000010:2.0:1788971592.651486:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000800:00000200:3.0:1788971592.651491:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000100:00000200:2.0:1788971592.651507:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410a80 x1875872179053696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971608 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971592.651512:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971592.651518:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1da00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971592.651528:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971592.651544:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971592.651548:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1da00. 00000400:00000200:2.0:1788971592.651551:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971592.651555:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971592.651558:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0 00000400:00000010:2.0:1788971592.651560:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0. 00000100:00000200:2.0:1788971592.651565:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179053568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971608 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971592.654818:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.654827:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971592.654835:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.654839:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.654845:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.654849:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06400 00000400:00000200:2.0:1788971592.654854:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.654860:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xbe5 [1] + 224 00000400:00000200:2.0:1788971592.654873:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971592.654878:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971592.654890:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179053568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971608 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971592.655133:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:3.0:1788971592.655139:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788971592.655142:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971592.655149:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179053568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971608 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971592.656219:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179053568/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971608 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971592.656250:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548e00. 02000000:00000010:3.0:1788971592.656255:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700. 00000100:00000010:3.0:1788971592.656260:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 00000400:00000200:2.0:1788971592.656799:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.656805:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971592.656813:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.656817:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.656822:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.656825:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06480 00000400:00000200:2.0:1788971592.656829:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971592.656835:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xbd5 [1] + 224 00000400:00000200:2.0:1788971592.656848:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971592.656853:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971592.656861:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410a80 x1875872179053696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971608 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971592.656896:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c60. 00000400:00000200:2.0:1788971592.656900:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:2.0:1788971592.656902:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788971592.656908:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410a80 x1875872179053696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971608 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971592.656929:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410a80 x1875872179053696/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971608 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971592.656959:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:2.0:1788971592.656964:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d000. 00000100:00000010:2.0:1788971592.656967:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410a80. 00000020:00000010:0.0:1788971592.945677:0:108147:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971592.945760:0:108147:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000100:00000010:3.0:1788971593.351503:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7780. 00000100:00000010:2.0:1788971593.351816:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7cc0. 00000020:00000010:1.0:1788971593.978527:0:108151:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308b70. 00000020:00000010:1.0:1788971593.978613:0:108151:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308b70. 00000020:00000010:3.0:1788971595.026071:0:108155:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971595.026162:0:108155:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971596.058173:0:108159:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971596.058287:0:108159:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971597.102297:0:108163:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971597.105752:0:108163:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000100:00000010:2.0:1788971597.767456:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410a80. 02000000:00000010:2.0:1788971597.767487:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d000. 00000100:00000010:2.0:1788971597.767528:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788971597.767534:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0df00. 02000000:00000010:2.0:1788971597.767571:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:2.0:1788971597.767590:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971597.767600:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c60. 00000400:00000010:2.0:1788971597.767607:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788971597.767614:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053952, portal 10 00000400:00000010:2.0:1788971597.767621:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971597.767624:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179053952, offset 0 00000400:00000200:2.0:1788971597.767632:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971597.767640:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.767648:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971597.767652:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971597.767656:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971597.767659:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971597.767662:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971597.767666:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [207:207:256:4294967295] 192.168.203.124@tcp LPNI seq info [207:207:8:4294967295] 00000400:00000200:2.0:1788971597.767670:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.767678:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971597.767684:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971597.767690:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700. 00000800:00000200:2.0:1788971597.767710:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971597.767717:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971597.767722:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971597.767737:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971597.767741:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7cc0. 00000100:00000010:2.0:1788971597.767751:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f77e0. 02000000:00000010:3.0:1788971597.767950:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549200. 00000400:00000200:3.0:1788971597.767956:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971597.767959:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971597.767963:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971597.767966:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179053824, portal 25 00000400:00000010:3.0:1788971597.767970:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081770. 00000100:00000200:3.0:1788971597.767972:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179053824, offset 0 00000400:00000200:3.0:1788971597.767985:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971597.767989:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971597.767993:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971597.767996:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971597.767998:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971597.768001:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971597.768004:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971597.768006:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [208:208:255:4294967295] 192.168.203.124@tcp LPNI seq info [208:208:7:4294967295] 00000400:00000200:3.0:1788971597.768011:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971597.768017:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971597.768021:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971597.768025:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d400. 00000800:00000200:3.0:1788971597.768029:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971597.768034:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971597.768038:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971597.768042:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971597.768162:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971597.768166:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700. 00000400:00000200:2.0:1788971597.768172:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971597.768178:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971597.768183:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:2.0:1788971597.768186:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971597.768194:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179053952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971597.768265:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971597.768268:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d400. 00000400:00000200:2.0:1788971597.768272:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971597.768275:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971597.768278:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081770 00000400:00000010:2.0:1788971597.768279:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081770. 00000100:00000200:2.0:1788971597.768284:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410a80 x1875872179053824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971597.772077:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.772086:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971597.772094:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.772099:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.772103:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.772108:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06500 00000400:00000200:2.0:1788971597.772113:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.772119:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc05 [1] + 224 00000400:00000200:2.0:1788971597.772133:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971597.772152:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971597.772166:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410a80 x1875872179053824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971597.772264:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:3.0:1788971597.772270:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788971597.772273:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971597.772280:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410a80 x1875872179053824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971597.772288:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410a80 x1875872179053824/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971597.772341:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549200. 02000000:00000010:3.0:1788971597.772348:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d000. 00000100:00000010:3.0:1788971597.772352:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410a80. 00000400:00000200:2.0:1788971597.773091:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.773096:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971597.773102:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.773105:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.773108:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.773111:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06580 00000400:00000200:2.0:1788971597.773115:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971597.773120:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xbf5 [1] + 224 00000400:00000200:2.0:1788971597.773129:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971597.773134:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971597.773142:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179053952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971597.773222:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c60. 00000400:00000200:3.0:1788971597.773228:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:3.0:1788971597.773230:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:3.0:1788971597.773237:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179053952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971597.773261:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179053952/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971597.773283:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:3.0:1788971597.773289:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0df00. 00000100:00000010:3.0:1788971597.773291:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000020:00000010:0.0:1788971598.146471:0:108167:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971598.146566:0:108167:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000100:00000010:2.0:1788971598.471421:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7cc0. 00000100:00000010:3.0:1788971598.471967:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f77e0. 00000020:00000010:3.0:1788971599.188630:0:108171:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971599.188717:0:108171:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000020:00000010:2.0:1788971600.248885:0:108176:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb461667a0. 00000020:00000010:2.0:1788971600.248984:0:108176:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb461667a0. 00000020:00000010:0.0:1788971601.274238:0:108180:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94a60. 00000020:00000010:0.0:1788971601.274355:0:108180:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94a60. 00000020:00000010:3.0:1788971601.306764:0:108181:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44cca020. 00000020:00000010:3.0:1788971601.307761:0:108181:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44cca020. 00000100:00000010:2.0:1788971602.887931:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788971602.887956:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700. 00000100:00000010:2.0:1788971602.887990:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788971602.887996:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d400. 02000000:00000010:3.0:1788971602.890650:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548200. 00000400:00000200:3.0:1788971602.890675:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971602.890681:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971602.890690:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971602.890700:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054080, portal 25 00000400:00000010:3.0:1788971602.890707:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081dd0. 00000100:00000200:3.0:1788971602.890711:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179054080, offset 0 00000400:00000200:3.0:1788971602.890721:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971602.890733:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971602.890744:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971602.890751:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971602.890756:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971602.890761:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971602.890764:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971602.890769:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [209:209:256:4294967295] 192.168.203.124@tcp LPNI seq info [209:209:8:4294967295] 00000400:00000200:3.0:1788971602.890773:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971602.890783:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971602.891080:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971602.891088:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d300. 00000800:00000200:3.0:1788971602.891115:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971602.891125:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971602.891132:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971602.891167:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971602.891174:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a120. 00000100:00000010:3.0:1788971602.891190:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a8a0. 02000000:00000010:3.0:1788971602.891221:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549800. 00000400:00000200:3.0:1788971602.891226:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971602.891229:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e783c8. 00000400:00000010:3.0:1788971602.891234:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971602.891238:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054208, portal 10 00000400:00000010:3.0:1788971602.891244:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480817f8. 00000100:00000200:3.0:1788971602.891247:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179054208, offset 0 00000400:00000200:3.0:1788971602.891253:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971602.891258:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971602.891262:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971602.891266:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971602.891268:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971602.891271:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971602.891274:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971602.891277:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [210:210:255:4294967295] 192.168.203.124@tcp LPNI seq info [210:210:7:4294967295] 00000400:00000200:3.0:1788971602.891281:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971602.891288:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971602.891293:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971602.891297:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:2.0:1788971602.891725:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971602.891731:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d300. 00000400:00000200:2.0:1788971602.891739:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971602.891746:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971602.891753:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081dd0 00000400:00000010:2.0:1788971602.891756:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081dd0. 00000100:00000200:2.0:1788971602.891768:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179054080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971602.892662:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971602.892671:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971602.892676:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971602.892706:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971602.892954:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971602.892960:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:2.0:1788971602.892965:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971602.892971:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971602.892978:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480817f8 00000400:00000010:2.0:1788971602.892981:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480817f8. 00000100:00000200:2.0:1788971602.892992:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179054208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971602.897237:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897250:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971602.897261:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897266:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897273:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897278:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06600 00000400:00000200:2.0:1788971602.897284:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897291:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc15 [1] + 224 00000400:00000200:2.0:1788971602.897355:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971602.897360:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971602.897382:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179054080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971602.897528:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897532:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971602.897536:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897539:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897542:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897545:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06680 00000400:00000200:2.0:1788971602.897549:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971602.897553:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc25 [1] + 224 00000400:00000200:2.0:1788971602.897562:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971602.897585:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971602.897594:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179054208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971602.899506:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:3.0:1788971602.899523:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:3.0:1788971602.899546:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971602.899563:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179054080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971602.899575:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179054080/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971602.899613:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548200. 02000000:00000010:3.0:1788971602.899621:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700. 00000100:00000010:3.0:1788971602.899629:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000400:00000010:3.0:1788971602.899642:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e783c8. 00000400:00000200:3.0:1788971602.899644:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:3.0:1788971602.899646:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971602.899652:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179054208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971602.899661:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179054208/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971602.899683:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549800. 02000000:00000010:3.0:1788971602.899687:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d400. 00000100:00000010:3.0:1788971602.899689:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 00000100:00000010:3.0:1788971603.591788:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a8a0. 00000100:00000010:2.0:1788971603.592179:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a120. 00000100:00000010:2.0:1788971608.007559:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788971608.007592:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d300. 00000100:00000010:2.0:1788971608.008015:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788971608.008024:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d400. 02000000:00000010:3.0:1788971608.011274:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548400. 00000400:00000200:3.0:1788971608.011297:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971608.011363:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e783c8. 00000400:00000010:3.0:1788971608.011373:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971608.011383:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054336, portal 25 00000400:00000010:3.0:1788971608.011400:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971608.011404:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179054336, offset 0 00000400:00000200:3.0:1788971608.011415:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971608.011426:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971608.011447:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971608.011454:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971608.011480:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971608.011485:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971608.011488:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971608.011493:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [211:211:256:4294967295] 192.168.203.124@tcp LPNI seq info [211:211:8:4294967295] 00000400:00000200:3.0:1788971608.011498:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971608.011510:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971608.011518:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971608.011524:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d500. 00000800:00000200:3.0:1788971608.011648:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971608.011660:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971608.011665:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971608.011690:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971608.011697:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a8a0. 00000100:00000010:3.0:1788971608.011714:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859aae0. 02000000:00000010:3.0:1788971608.011744:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549c00. 00000400:00000200:3.0:1788971608.011749:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971608.011752:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971608.011756:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971608.011759:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054464, portal 10 00000400:00000010:3.0:1788971608.011765:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081b28. 00000100:00000200:3.0:1788971608.011768:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179054464, offset 0 00000400:00000200:3.0:1788971608.011773:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971608.011777:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971608.011780:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971608.011782:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971608.011784:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971608.011786:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971608.011788:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971608.011791:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [212:212:255:4294967295] 192.168.203.124@tcp LPNI seq info [212:212:7:4294967295] 00000400:00000200:3.0:1788971608.011795:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971608.011799:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971608.011811:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971608.011814:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d600. 00000800:00000200:3.0:1788971608.011818:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971608.011822:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971608.011825:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971608.011831:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971608.011895:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971608.011901:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d500. 00000400:00000200:2.0:1788971608.011908:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971608.011914:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971608.011920:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:2.0:1788971608.011922:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:2.0:1788971608.011933:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179054336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971608.011962:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971608.011964:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d600. 00000400:00000200:2.0:1788971608.011967:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971608.011970:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971608.011973:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081b28 00000400:00000010:2.0:1788971608.011975:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081b28. 00000100:00000200:2.0:1788971608.011979:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179054464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971608.014231:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.014244:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971608.014254:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.014259:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.014266:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.014271:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06700 00000400:00000200:2.0:1788971608.014277:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.014418:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc35 [1] + 224 00000400:00000200:2.0:1788971608.014439:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971608.014444:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971608.014461:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179054336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971624 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971608.014525:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e783c8. 00000400:00000200:3.0:1788971608.014532:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:3.0:1788971608.014535:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971608.014543:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179054336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971624 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971608.014555:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179054336/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971624 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971608.014589:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548400. 02000000:00000010:3.0:1788971608.014594:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d300. 00000100:00000010:3.0:1788971608.014598:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 00000400:00000200:2.0:1788971608.021055:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.021068:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971608.021079:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.021084:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.021091:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.021097:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06780 00000400:00000200:2.0:1788971608.021103:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971608.021111:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc45 [1] + 224 00000400:00000200:2.0:1788971608.021129:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971608.021134:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971608.021151:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179054464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971624 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971608.023179:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:3.0:1788971608.023194:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788971608.023198:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971608.023214:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179054464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971624 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971608.023229:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179054464/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971624 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971608.023273:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549c00. 02000000:00000010:3.0:1788971608.023279:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d400. 00000100:00000010:3.0:1788971608.023285:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 00000100:00000010:2.0:1788971608.711479:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a8a0. 00000100:00000010:3.0:1788971608.711711:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859aae0. 00000100:00000010:0.0:1788971613.128589:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788971613.128612:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788971613.128656:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788971613.128662:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55900. 02000000:00000010:1.0:1788971613.129459:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c400. 00000400:00000200:1.0:1788971613.129468:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971613.129472:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971613.129477:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971613.129482:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054592, portal 25 00000400:00000010:1.0:1788971613.129486:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788971613.129489:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179054592, offset 0 00000400:00000200:1.0:1788971613.129495:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971613.129508:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971613.129517:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971613.129521:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971613.129525:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971613.129529:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971613.129532:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971613.129536:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [213:213:256:4294967295] 192.168.203.124@tcp LPNI seq info [213:213:8:4294967295] 00000400:00000200:1.0:1788971613.129540:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971613.129550:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971613.129556:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971613.129561:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71700. 00000800:00000200:1.0:1788971613.129568:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971613.129576:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971613.129580:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971613.129601:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971613.129607:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fa20. 00000100:00000010:1.0:1788971613.129616:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fba0. 02000000:00000010:1.0:1788971613.129662:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124ca00. 00000400:00000200:1.0:1788971613.129668:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971613.129670:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971613.129674:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788971613.129677:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054720, portal 10 00000400:00000010:1.0:1788971613.129682:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788971613.129684:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179054720, offset 0 00000400:00000200:1.0:1788971613.129689:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971613.129693:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971613.129696:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971613.129699:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971613.129702:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971613.129704:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971613.129706:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971613.129709:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [214:214:255:4294967295] 192.168.203.124@tcp LPNI seq info [214:214:7:4294967295] 00000400:00000200:1.0:1788971613.129713:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971613.129718:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971613.129722:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971613.129726:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71100. 00000800:00000200:1.0:1788971613.129730:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971613.129734:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971613.129736:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971613.129741:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971613.130011:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971613.130016:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71700. 00000400:00000200:2.0:1788971613.130023:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971613.130029:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971613.130035:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:2.0:1788971613.130037:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:2.0:1788971613.130046:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179054592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971613.130123:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971613.130126:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71100. 00000400:00000200:2.0:1788971613.130129:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971613.130132:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971613.130143:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:2.0:1788971613.130145:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:2.0:1788971613.130150:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179054720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971613.132948:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.132957:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971613.132966:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.132970:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.132975:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.132979:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06880 00000400:00000200:2.0:1788971613.132985:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.132991:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc5d [1] + 224 00000400:00000200:2.0:1788971613.133004:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971613.133009:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971613.133026:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179054720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971629 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971613.134465:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.134477:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971613.134484:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.134488:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.134492:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.134496:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06800 00000400:00000200:2.0:1788971613.134500:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971613.134505:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc55 [1] + 224 00000400:00000200:2.0:1788971613.134517:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971613.134521:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971613.134531:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179054592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971629 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971613.134773:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971613.134779:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588f68 00000400:00000010:1.0:1788971613.134782:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788971613.134789:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179054720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971629 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971613.134798:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179054720/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971629 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971613.134856:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971613.134860:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:1.0:1788971613.134862:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971613.134866:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179054592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971629 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971613.134872:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179054592/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971629 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971613.134887:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124ca00. 02000000:00000010:1.0:1788971613.134894:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55900. 00000100:00000010:1.0:1788971613.134899:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 02000000:00000010:1.0:1788971613.134903:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c400. 02000000:00000010:1.0:1788971613.134905:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788971613.134907:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000100:00000010:3.0:1788971613.831578:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fba0. 00000100:00000010:2.0:1788971613.831809:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fa20. 00000100:00000010:1.0:1788971618.247946:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971618.247969:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71200. 00000100:00000010:1.0:1788971618.248008:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6680. 02000000:00000010:1.0:1788971618.248013:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71300. 02000000:00000010:1.0:1788971618.248053:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124ca00. 00000400:00000200:1.0:1788971618.248063:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971618.248067:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971618.248073:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971618.248080:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054976, portal 10 00000400:00000010:1.0:1788971618.248087:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788971618.248091:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179054976, offset 0 00000400:00000200:1.0:1788971618.248100:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971618.248109:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971618.248116:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971618.248121:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971618.248125:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971618.248128:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971618.248145:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971618.248149:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [215:215:256:4294967295] 192.168.203.124@tcp LPNI seq info [215:215:8:4294967295] 00000400:00000200:1.0:1788971618.248154:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971618.248163:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971618.248169:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971618.248175:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71600. 00000800:00000200:1.0:1788971618.248182:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971618.248190:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971618.248195:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71600 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:0.0:1788971618.248198:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aae00. 00000400:00000200:0.0:1788971618.248206:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971618.248209:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971618.248214:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971618.248217:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179054848, portal 25 00000400:00000200:1.0:1788971618.248223:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971618.248231:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000010:1.0:1788971618.248232:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fb40. 00000100:00000200:0.0:1788971618.248236:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179054848, offset 0 00000400:00000200:0.0:1788971618.248244:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971618.248249:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971618.248250:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f480. 00000400:00000200:0.0:1788971618.248255:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971618.248259:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971618.248262:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971618.248264:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971618.248267:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971618.248270:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [216:216:255:4294967295] 192.168.203.124@tcp LPNI seq info [216:216:7:4294967295] 00000400:00000200:0.0:1788971618.248275:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971618.248284:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971618.248289:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971618.248333:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500. 00000800:00000200:0.0:1788971618.248340:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971618.248345:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971618.248349:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971618.248354:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971618.248408:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971618.248413:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71600. 00000400:00000200:2.0:1788971618.248419:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971618.248425:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971618.248429:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588f68 00000400:00000010:2.0:1788971618.248432:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588f68. 00000100:00000200:2.0:1788971618.248440:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6680 x1875872179054976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971618.248515:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971618.248518:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500. 00000400:00000200:2.0:1788971618.248522:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971618.248525:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971618.248528:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:2.0:1788971618.248530:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:2.0:1788971618.248534:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179054848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971618.252490:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.252500:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971618.252507:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.252512:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.252517:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.252521:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06900 00000400:00000200:2.0:1788971618.252527:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.252532:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc6d [1] + 224 00000400:00000200:2.0:1788971618.252545:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971618.252549:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971618.252563:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179054848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971634 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971618.252869:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971618.252875:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971618.252877:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971618.252884:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179054848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971634 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971618.252894:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179054848/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971634 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971618.252922:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aae00. 02000000:00000010:0.0:1788971618.252928:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71200. 00000100:00000010:0.0:1788971618.252933:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 00000400:00000200:2.0:1788971618.253262:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.253266:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971618.253272:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.253289:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.253292:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.253349:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06980 00000400:00000200:2.0:1788971618.253354:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971618.253358:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc65 [1] + 224 00000400:00000200:2.0:1788971618.253370:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971618.253373:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971618.253380:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6680 x1875872179054976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971634 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971618.253519:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971618.253525:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:1.0:1788971618.253527:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971618.253533:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6680 x1875872179054976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971634 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971618.253543:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6680 x1875872179054976/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971634 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971618.253562:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124ca00. 02000000:00000010:1.0:1788971618.253567:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71300. 00000100:00000010:1.0:1788971618.253570:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6680. 00000100:00000010:3.0:1788971618.951788:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f480. 00000100:00000010:2.0:1788971618.951917:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fb40. 00000100:00000010:1.0:1788971623.367694:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6680. 02000000:00000010:1.0:1788971623.367719:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71300. 00000100:00000010:1.0:1788971623.367761:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971623.367767:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71f00. 02000000:00000010:1.0:1788971623.367796:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124ca00. 00000400:00000200:1.0:1788971623.367804:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971623.367809:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971623.367828:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971623.367836:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055232, portal 10 00000400:00000010:1.0:1788971623.367843:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588b28. 00000100:00000200:1.0:1788971623.367846:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179055232, offset 0 00000400:00000200:1.0:1788971623.367855:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971623.367864:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971623.367872:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971623.367878:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971623.367882:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971623.367885:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971623.367888:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971623.367892:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [217:217:256:4294967295] 192.168.203.124@tcp LPNI seq info [217:217:8:4294967295] 00000400:00000200:1.0:1788971623.367896:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971623.367905:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971623.367911:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971623.367916:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71a00. 00000800:00000200:1.0:1788971623.367923:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971623.367930:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971623.367935:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971623.367960:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971623.367966:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fc00. 00000100:00000010:1.0:1788971623.367980:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f060. 02000000:00000010:1.0:1788971623.368013:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124d200. 00000400:00000200:1.0:1788971623.368019:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971623.368022:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971623.368026:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588a18. 00000100:00000200:1.0:1788971623.368029:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055104, portal 25 00000400:00000010:1.0:1788971623.368033:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588990. 00000100:00000200:1.0:1788971623.368035:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179055104, offset 0 00000400:00000200:1.0:1788971623.368039:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971623.368043:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971623.368053:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971623.368057:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971623.368060:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971623.368062:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971623.368065:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971623.368068:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [218:218:255:4294967295] 192.168.203.124@tcp LPNI seq info [218:218:7:4294967295] 00000400:00000200:1.0:1788971623.368072:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971623.368109:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971623.368114:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71a00. 00000400:00000200:2.0:1788971623.368120:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971623.368126:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:1.0:1788971623.368200:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971623.368208:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971623.368212:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71a00. 00000800:00000200:1.0:1788971623.368215:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971623.368219:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971623.368222:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971623.368226:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.370874:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588b28 00000400:00000010:2.0:1788971623.370884:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588b28. 00000100:00000200:2.0:1788971623.370901:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179055232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971623.371031:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971623.371035:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71a00. 00000400:00000200:2.0:1788971623.371040:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971623.371045:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971623.371048:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588990 00000400:00000010:2.0:1788971623.371051:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588990. 00000100:00000200:2.0:1788971623.371056:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6680 x1875872179055104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971623.374617:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374627:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971623.374635:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374640:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374646:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374663:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06a80 00000400:00000200:2.0:1788971623.374668:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374674:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc75 [1] + 224 00000400:00000200:2.0:1788971623.374686:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971623.374691:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971623.374705:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179055232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971639 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971623.374746:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374750:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971623.374754:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374756:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374759:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374762:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06a00 00000400:00000200:2.0:1788971623.374765:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971623.374769:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc7d [1] + 224 00000400:00000200:2.0:1788971623.374778:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971623.374781:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971623.374787:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6680 x1875872179055104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971639 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971623.375255:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971623.375262:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588a18 00000400:00000010:1.0:1788971623.375264:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588a18. 00000100:00000200:1.0:1788971623.375271:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6680 x1875872179055104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971639 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971623.375291:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6680 x1875872179055104/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971639 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971623.375341:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971623.375345:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:1.0:1788971623.375347:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971623.375352:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179055232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971639 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971623.375359:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179055232/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971639 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971623.375379:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124d200. 02000000:00000010:1.0:1788971623.375394:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71300. 00000100:00000010:1.0:1788971623.375399:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6680. 02000000:00000010:1.0:1788971623.375403:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124ca00. 02000000:00000010:1.0:1788971623.375407:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71f00. 00000100:00000010:1.0:1788971623.375409:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 00000100:00000010:3.0:1788971624.071405:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f060. 00000100:00000010:2.0:1788971624.072448:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fc00. 00000100:00000010:0.0:1788971628.487526:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788971628.487550:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 02000000:00000010:1.0:1788971628.487634:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124d200. 00000400:00000200:1.0:1788971628.487644:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971628.487649:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971628.487655:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971628.487661:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055360, portal 25 00000400:00000010:1.0:1788971628.487665:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588a18. 00000100:00000200:1.0:1788971628.487667:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179055360, offset 0 00000400:00000200:1.0:1788971628.487674:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971628.487680:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971628.487686:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971628.487689:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971628.487692:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971628.487695:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971628.487697:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971628.487708:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [219:219:256:4294967295] 192.168.203.124@tcp LPNI seq info [219:219:8:4294967295] 00000400:00000200:1.0:1788971628.487710:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971628.487717:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971628.487721:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971628.487725:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71f00. 00000800:00000200:1.0:1788971628.487730:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971628.487735:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971628.487739:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971628.487752:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971628.487756:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f060. 00000100:00000010:1.0:1788971628.487764:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fb40. 00000800:00000200:2.0:1788971628.487880:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971628.487883:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71f00. 00000400:00000200:2.0:1788971628.487887:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971628.487890:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971628.487895:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588a18 00000400:00000010:2.0:1788971628.487897:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588a18. 00000100:00000200:2.0:1788971628.487906:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179055360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788971628.490394:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788971628.490416:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 02000000:00000010:1.0:1788971628.492239:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c000. 00000400:00000200:1.0:1788971628.492276:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971628.492281:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971628.492288:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971628.492294:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055488, portal 10 00000400:00000010:1.0:1788971628.492315:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588d48. 00000100:00000200:1.0:1788971628.492319:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179055488, offset 0 00000400:00000200:1.0:1788971628.492328:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971628.492335:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971628.492343:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971628.492348:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971628.492352:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971628.492355:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971628.492363:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971628.492367:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [220:220:256:4294967295] 192.168.203.124@tcp LPNI seq info [220:220:8:4294967295] 00000400:00000200:1.0:1788971628.492370:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971628.492379:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971628.492386:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971628.492391:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71300. 00000800:00000200:1.0:1788971628.492398:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971628.492405:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971628.492409:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971628.492423:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971628.492597:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971628.492603:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971628.492615:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971628.492619:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971628.492623:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971628.492627:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06b00 00000400:00000200:3.0:1788971628.492632:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971628.492637:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xc85 [1] + 224 00000800:00000200:2.0:1788971628.492683:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971628.492694:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71300. 00000400:00000200:2.0:1788971628.492708:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971628.492714:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971628.492719:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588d48 00000400:00000010:2.0:1788971628.492721:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588d48. 00000100:00000200:2.0:1788971628.492730:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179055488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971628.492806:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971628.492811:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971628.492819:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179055360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971644 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971628.493272:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971628.493277:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:1.0:1788971628.493279:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971628.493286:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179055360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971644 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971628.493295:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179055360/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971644 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971628.493347:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124d200. 02000000:00000010:1.0:1788971628.493361:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788971628.493365:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 00000400:00000200:2.0:1788971628.498007:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971628.498017:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971628.498025:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971628.498030:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971628.498035:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971628.498050:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06b80 00000400:00000200:2.0:1788971628.498055:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971628.498061:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc8d [1] + 224 00000400:00000200:2.0:1788971628.498076:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971628.498080:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971628.498094:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179055488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971644 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971628.498590:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971628.498596:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971628.498599:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971628.498606:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179055488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971644 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971628.498615:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179055488/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971644 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971628.498645:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c000. 02000000:00000010:1.0:1788971628.498652:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788971628.498657:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000100:00000010:3.0:1788971629.191753:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fb40. 00000100:00000010:2.0:1788971629.192376:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f060. 00000100:00000010:3.0:1788971633.607564:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480. 02000000:00000010:3.0:1788971633.607588:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d100. 00000100:00000010:3.0:1788971633.607629:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4000. 02000000:00000010:3.0:1788971633.607635:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d600. 02000000:00000010:2.0:1788971633.607733:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:2.0:1788971633.607743:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971633.607748:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788971633.607754:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971633.607760:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055744, portal 10 00000400:00000010:2.0:1788971633.607767:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788971633.607770:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179055744, offset 0 00000400:00000200:2.0:1788971633.607779:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971633.607787:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.607794:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971633.607799:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971633.607817:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971633.607821:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971633.607824:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971633.607827:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [221:221:256:4294967295] 192.168.203.124@tcp LPNI seq info [221:221:8:4294967295] 00000400:00000200:2.0:1788971633.607832:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.607841:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971633.607848:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971633.607853:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700. 00000800:00000200:2.0:1788971633.607860:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971633.607867:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971633.607872:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971633.607892:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971633.607897:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7960. 00000100:00000010:2.0:1788971633.607909:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7cc0. 02000000:00000010:2.0:1788971633.607934:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5200. 00000400:00000200:2.0:1788971633.607938:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971633.607941:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c60. 00000400:00000010:2.0:1788971633.607945:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788971633.607947:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055616, portal 25 00000400:00000010:2.0:1788971633.607951:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0e58. 00000100:00000200:2.0:1788971633.607953:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179055616, offset 0 00000400:00000200:2.0:1788971633.607958:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971633.607961:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.607965:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971633.607968:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971633.607970:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971633.607972:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971633.607974:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971633.607977:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [222:222:255:4294967295] 192.168.203.124@tcp LPNI seq info [222:222:7:4294967295] 00000400:00000200:2.0:1788971633.607981:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.607987:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971633.607998:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971633.608002:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0df00. 00000800:00000200:2.0:1788971633.608006:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971633.608010:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971633.608013:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971633.608018:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971633.608126:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971633.608130:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700. 00000400:00000200:3.0:1788971633.608136:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971633.608141:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971633.608147:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:3.0:1788971633.608149:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:3.0:1788971633.608167:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4000 x1875872179055744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971649 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971633.608253:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971633.608256:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0df00. 00000400:00000200:3.0:1788971633.608259:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971633.608262:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971633.608265:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0e58 00000400:00000010:3.0:1788971633.608275:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0e58. 00000100:00000200:3.0:1788971633.608281:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7480 x1875872179055616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971649 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971633.614348:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.614357:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971633.614377:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.614383:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.614388:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.614393:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06c00 00000400:00000200:2.0:1788971633.614398:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971633.614405:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xca5 [1] + 224 00000400:00000200:2.0:1788971633.614420:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971633.614424:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971633.614438:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7480 x1875872179055616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971649 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971633.614668:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c60. 00000400:00000200:3.0:1788971633.614674:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:3.0:1788971633.614677:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:3.0:1788971633.614684:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7480 x1875872179055616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971649 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971633.614694:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7480 x1875872179055616/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971649 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971633.614721:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5200. 02000000:00000010:3.0:1788971633.614727:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d100. 00000100:00000010:3.0:1788971633.614732:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480. 00000400:00000200:3.0:1788971633.622574:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971633.622584:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971633.622592:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971633.622597:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971633.622602:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971633.622605:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06c80 00000400:00000200:3.0:1788971633.622610:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971633.622615:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xc95 [1] + 224 00000400:00000200:3.0:1788971633.622628:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971633.622632:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971633.622647:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4000 x1875872179055744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971649 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971633.622910:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:2.0:1788971633.622916:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:2.0:1788971633.622919:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971633.622926:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4000 x1875872179055744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971649 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971633.622935:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4000 x1875872179055744/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971649 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971633.622970:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:2.0:1788971633.622976:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d600. 00000100:00000010:2.0:1788971633.622981:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4000. 00000100:00000010:3.0:1788971634.311970:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7cc0. 00000100:00000010:2.0:1788971634.312216:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7960. 00000100:00000010:3.0:1788971638.727565:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480. 02000000:00000010:3.0:1788971638.727596:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d100. 00000100:00000010:3.0:1788971638.727650:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788971638.727657:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d500. 02000000:00000010:2.0:1788971638.728181:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:2.0:1788971638.728195:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971638.728201:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788971638.728210:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971638.728220:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179055872, portal 25 00000400:00000010:2.0:1788971638.728226:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788971638.728229:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179055872, offset 0 00000400:00000200:2.0:1788971638.728241:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971638.728252:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971638.728262:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971638.728269:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971638.728273:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971638.728278:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971638.728281:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971638.728285:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [223:223:256:4294967295] 192.168.203.124@tcp LPNI seq info [223:223:8:4294967295] 00000400:00000200:2.0:1788971638.728289:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971638.728347:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971638.728354:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971638.728361:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d000. 00000800:00000200:2.0:1788971638.728369:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971638.728378:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971638.728384:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971638.728407:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971638.728413:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7960. 00000100:00000010:2.0:1788971638.728427:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7e40. 02000000:00000010:2.0:1788971638.728452:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4200. 00000400:00000200:2.0:1788971638.728457:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971638.728459:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788971638.728464:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788971638.728466:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056000, portal 10 00000400:00000010:2.0:1788971638.728473:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788971638.728486:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179056000, offset 0 00000400:00000200:2.0:1788971638.728492:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971638.728496:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971638.728499:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971638.728502:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971638.728505:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971638.728507:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971638.728510:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971638.728513:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [224:224:255:4294967295] 192.168.203.124@tcp LPNI seq info [224:224:7:4294967295] 00000400:00000200:2.0:1788971638.728517:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971638.728522:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971638.728526:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971638.728531:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0df00. 00000800:00000200:2.0:1788971638.728535:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971638.728539:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971638.728542:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971638.728548:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971638.729475:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971638.729483:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d000. 00000400:00000200:3.0:1788971638.729492:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971638.729500:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971638.729508:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:3.0:1788971638.729512:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:3.0:1788971638.729524:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7480 x1875872179055872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971638.730250:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971638.730257:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0df00. 00000400:00000200:3.0:1788971638.730261:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971638.730267:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971638.730273:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:3.0:1788971638.730275:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:3.0:1788971638.730284:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179056000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971654 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971638.732880:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.732893:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971638.732903:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.732909:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.732916:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.732920:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06d80 00000400:00000200:3.0:1788971638.732927:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.732935:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xcc5 [1] + 224 00000400:00000200:3.0:1788971638.732956:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971638.732961:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971638.732980:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179056000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971654 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971638.733491:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.733496:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971638.733502:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.733527:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.733538:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.733542:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06d00 00000400:00000200:3.0:1788971638.733546:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971638.733552:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xcb5 [1] + 224 00000400:00000200:3.0:1788971638.733563:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971638.733567:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971638.733576:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7480 x1875872179055872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971654 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971638.733669:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:2.0:1788971638.733677:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:2.0:1788971638.733679:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788971638.733688:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179056000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971654 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971638.733697:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179056000/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971654 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971638.733815:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:2.0:1788971638.733819:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:2.0:1788971638.733821:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971638.733826:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7480 x1875872179055872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971654 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971638.733833:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7480 x1875872179055872/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971654 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971638.733862:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4200. 02000000:00000010:2.0:1788971638.733871:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d500. 00000100:00000010:2.0:1788971638.733878:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 02000000:00000010:2.0:1788971638.733882:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:2.0:1788971638.733885:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d100. 00000100:00000010:2.0:1788971638.733888:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480. 00000100:00000010:3.0:1788971639.431468:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7e40. 00000100:00000010:2.0:1788971639.431826:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7960. 00000100:00000010:2.0:1788971643.847455:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788971643.847495:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700. 00000100:00000010:2.0:1788971643.847745:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788971643.847753:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d400. 02000000:00000010:2.0:1788971643.847805:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:2.0:1788971643.847817:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971643.847822:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788971643.847831:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971643.847839:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056256, portal 10 00000400:00000010:2.0:1788971643.847847:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788971643.847850:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179056256, offset 0 00000400:00000200:2.0:1788971643.847860:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971643.847871:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971643.847880:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971643.847887:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971643.847891:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971643.847896:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971643.847898:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971643.847912:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [225:225:256:4294967295] 192.168.203.124@tcp LPNI seq info [225:225:8:4294967295] 00000400:00000200:2.0:1788971643.847917:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971643.847928:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971643.847934:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971643.847939:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d300. 00000800:00000200:2.0:1788971643.847963:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971643.847972:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971643.847977:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971643.847993:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971643.848000:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7960. 02000000:00000010:3.0:1788971643.848234:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548a00. 00000400:00000200:3.0:1788971643.848241:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971643.848244:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78840. 00000400:00000010:3.0:1788971643.848250:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971643.848253:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056128, portal 25 00000400:00000010:3.0:1788971643.848257:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971643.848259:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179056128, offset 0 00000400:00000200:3.0:1788971643.848269:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971643.848273:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.848277:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971643.848280:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971643.848282:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971643.848285:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971643.848287:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971643.848290:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [226:226:255:4294967295] 192.168.203.124@tcp LPNI seq info [226:226:7:4294967295] 00000400:00000200:3.0:1788971643.848294:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.848338:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971643.848343:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971643.848348:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:3.0:1788971643.848352:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971643.848362:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971643.848366:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971643.848370:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971643.848425:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971643.848429:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d300. 00000400:00000200:3.0:1788971643.848437:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971643.848442:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971643.848448:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:3.0:1788971643.848451:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:3.0:1788971643.848462:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179056256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971643.848565:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971643.848568:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:3.0:1788971643.848572:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971643.848576:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971643.848578:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:3.0:1788971643.848580:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971643.848585:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179056128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971659 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788971643.848736:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f71e0. 00000400:00000200:3.0:1788971643.850808:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.850819:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971643.850827:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.850833:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.850838:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.850843:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06e00 00000400:00000200:3.0:1788971643.850849:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.850856:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xce5 [1] + 224 00000400:00000200:3.0:1788971643.850870:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971643.850875:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971643.850891:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179056128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971659 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971643.851370:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78840. 00000400:00000200:2.0:1788971643.851377:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:2.0:1788971643.851380:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:2.0:1788971643.851388:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179056128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971659 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971643.851398:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179056128/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971659 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971643.851433:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548a00. 02000000:00000010:2.0:1788971643.851441:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700. 00000100:00000010:2.0:1788971643.851447:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000400:00000200:3.0:1788971643.853502:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.853513:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971643.853523:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.853529:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.853535:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.853540:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06e80 00000400:00000200:3.0:1788971643.853546:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971643.853554:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xcd5 [1] + 224 00000400:00000200:3.0:1788971643.853572:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971643.853577:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971643.853593:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179056256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971659 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971643.853802:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:2.0:1788971643.853809:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:2.0:1788971643.853812:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971643.853820:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179056256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971659 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971643.853830:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179056256/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971659 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971643.853867:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:2.0:1788971643.853874:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d400. 00000100:00000010:2.0:1788971643.853879:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000100:00000010:3.0:1788971644.551671:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f71e0. 00000100:00000010:2.0:1788971644.551775:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7960. 00000100:00000010:1.0:1788971648.967725:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971648.967747:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71200. 00000100:00000010:1.0:1788971648.967789:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6680. 02000000:00000010:1.0:1788971648.967794:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49d71600. 02000000:00000010:1.0:1788971648.967829:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124cc00. 00000400:00000200:1.0:1788971648.967839:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971648.967843:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971648.967851:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971648.967857:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056512, portal 10 00000400:00000010:1.0:1788971648.967865:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971648.967868:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179056512, offset 0 00000400:00000200:1.0:1788971648.967877:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971648.967886:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971648.967894:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971648.967900:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971648.967904:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971648.967908:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971648.967910:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971648.967914:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [227:227:256:4294967295] 192.168.203.124@tcp LPNI seq info [227:227:8:4294967295] 00000400:00000200:1.0:1788971648.967945:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971648.967956:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971648.967962:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971648.967967:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71100. 00000800:00000200:1.0:1788971648.967974:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971648.967982:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971648.967997:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971648.968020:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971648.968025:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f480. 00000100:00000010:1.0:1788971648.968089:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fa20. 02000000:00000010:1.0:1788971648.968117:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124de00. 00000400:00000200:1.0:1788971648.968121:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971648.968123:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971648.968126:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788971648.968129:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056384, portal 25 00000400:00000010:1.0:1788971648.968132:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588e58. 00000100:00000200:1.0:1788971648.968134:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179056384, offset 0 00000400:00000200:1.0:1788971648.968138:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971648.968142:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971648.968145:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971648.968148:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971648.968150:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971648.968152:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971648.968154:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971648.968156:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [228:228:255:4294967295] 192.168.203.124@tcp LPNI seq info [228:228:7:4294967295] 00000400:00000200:1.0:1788971648.968159:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971648.968164:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971648.968168:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971648.968171:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71700. 00000800:00000200:1.0:1788971648.968174:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971648.968184:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971648.968186:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000200:1.0:1788971648.968187:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71700 type 1, nob 320 niov 1 nkiov 1 00000800:00000010:3.0:1788971648.968190:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71100. 00000400:00000200:1.0:1788971648.968192:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971648.968194:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971648.968198:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971648.968204:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:3.0:1788971648.968206:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:3.0:1788971648.968214:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6680 x1875872179056512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971648.968238:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971648.968240:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71700. 00000400:00000200:3.0:1788971648.968243:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971648.968246:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971648.968248:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588e58 00000400:00000010:3.0:1788971648.968250:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588e58. 00000100:00000200:3.0:1788971648.968254:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179056384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971664 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971648.973088:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973106:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971648.973116:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973137:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973142:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973146:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e06f00 00000400:00000200:2.0:1788971648.973151:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973158:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xcfd [1] + 224 00000400:00000200:2.0:1788971648.973170:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971648.973174:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971648.973188:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179056384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971664 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971648.973212:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973215:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971648.973218:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973220:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973222:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973225:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e06f80 00000400:00000200:2.0:1788971648.973228:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971648.973232:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xcf5 [1] + 224 00000400:00000200:2.0:1788971648.973240:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971648.973243:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971648.973248:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6680 x1875872179056512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971664 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971648.977187:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971648.977199:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588ee0 00000400:00000010:1.0:1788971648.977202:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788971648.977215:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179056384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971664 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971648.977227:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179056384/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971664 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971648.977253:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971648.977256:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971648.977257:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971648.977262:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6680 x1875872179056512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971664 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971648.977270:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6680 x1875872179056512/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971664 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971648.977327:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124de00. 02000000:00000010:1.0:1788971648.977337:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71200. 00000100:00000010:1.0:1788971648.977341:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971648.977345:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124cc00. 02000000:00000010:1.0:1788971648.977348:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49d71600. 00000100:00000010:1.0:1788971648.977350:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6680. 00000100:00000010:3.0:1788971649.671373:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fa20. 00000100:00000010:2.0:1788971649.671425:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f480. 00000100:00000010:0.0:1788971654.087881:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788971654.087902:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55900. 00000100:00000010:0.0:1788971654.087938:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788971654.087944:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55d00. 02000000:00000010:1.0:1788971654.088229:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124cc00. 00000400:00000200:1.0:1788971654.088240:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971654.088245:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971654.088252:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971654.088259:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056640, portal 25 00000400:00000010:1.0:1788971654.088264:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788971654.088268:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179056640, offset 0 00000400:00000200:1.0:1788971654.088290:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 02000000:00000010:0.0:1788971654.088370:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aae00. 00000400:00000200:0.0:1788971654.088376:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971654.088379:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971654.088383:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971654.088386:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056768, portal 10 00000400:00000010:0.0:1788971654.088391:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084330. 00000100:00000200:0.0:1788971654.088393:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179056768, offset 0 00000400:00000200:0.0:1788971654.088398:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971654.088421:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971654.088429:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971654.088434:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971654.088438:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971654.088442:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971654.088445:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971654.088448:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [229:229:256:4294967295] 192.168.203.124@tcp LPNI seq info [229:229:8:4294967295] 00000400:00000200:1.0:1788971654.088453:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971654.088462:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971654.088468:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971654.088473:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71600. 00000800:00000200:1.0:1788971654.088479:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971654.088486:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971654.088490:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971654.088511:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971654.088516:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fba0. 00000100:00000010:1.0:1788971654.088526:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f840. 00000800:00000200:3.0:1788971654.089675:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971654.089683:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71600. 00000400:00000200:3.0:1788971654.089689:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971654.089695:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971654.089701:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588ee0 00000400:00000010:3.0:1788971654.089713:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588ee0. 00000100:00000200:3.0:1788971654.089724:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179056640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971670 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971654.096371:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971654.096380:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971654.096389:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971654.096394:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971654.096398:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971654.096402:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07000 00000400:00000200:2.0:1788971654.096408:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971654.096414:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd05 [1] + 224 00000400:00000200:2.0:1788971654.096456:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971654.096460:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:0.0:1788971654.098433:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971654.098445:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971654.098451:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971654.098455:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971654.098458:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971654.098461:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971654.098464:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [230:230:256:4294967295] 192.168.203.124@tcp LPNI seq info [230:230:8:4294967295] 00000400:00000200:0.0:1788971654.098469:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971654.098480:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971654.098486:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971654.098496:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55b00. 00000800:00000200:0.0:1788971654.098504:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971654.098511:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971654.098516:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971654.098532:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971654.098737:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971654.098742:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55b00. 00000400:00000200:3.0:1788971654.098746:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971654.098751:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971654.098757:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084330 00000400:00000010:3.0:1788971654.098760:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084330. 00000100:00000200:3.0:1788971654.098770:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179056768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971670 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971654.099650:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179056640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971670 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971654.099758:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971654.099763:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971654.099766:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971654.099772:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179056640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971670 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971654.099780:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179056640/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971670 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971654.099804:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124cc00. 02000000:00000010:1.0:1788971654.099811:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55900. 00000100:00000010:1.0:1788971654.099815:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000400:00000200:3.0:1788971654.103291:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971654.103355:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971654.103368:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971654.103373:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971654.103380:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971654.103385:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07080 00000400:00000200:3.0:1788971654.103392:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971654.103400:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd0d [1] + 224 00000400:00000200:3.0:1788971654.103418:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971654.103424:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971654.103453:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179056768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971670 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971654.103597:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971654.103609:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971654.103636:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971654.103650:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179056768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971670 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971654.103662:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179056768/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971670 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971654.103696:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aae00. 02000000:00000010:0.0:1788971654.103701:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55d00. 00000100:00000010:0.0:1788971654.103706:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000100:00000010:3.0:1788971654.791451:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f840. 00000100:00000010:2.0:1788971654.791603:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fba0. 00000100:00000010:0.0:1788971659.207432:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788971659.207455:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55d00. 00000100:00000010:0.0:1788971659.207498:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788971659.207503:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 02000000:00000010:0.0:1788971659.207540:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aae00. 00000400:00000200:0.0:1788971659.207550:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971659.207554:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971659.207560:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971659.207567:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057024, portal 10 00000400:00000010:0.0:1788971659.207575:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480845d8. 00000100:00000200:0.0:1788971659.207578:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179057024, offset 0 00000400:00000200:0.0:1788971659.207587:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971659.207595:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971659.207602:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971659.207608:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971659.207612:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971659.207616:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971659.207618:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971659.207968:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [231:231:256:4294967295] 192.168.203.124@tcp LPNI seq info [231:231:8:4294967295] 00000400:00000200:0.0:1788971659.207974:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971659.207985:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971659.207991:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971659.208008:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55c00. 00000800:00000200:0.0:1788971659.208015:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971659.208023:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971659.208038:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971659.208061:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 02000000:00000010:1.0:1788971659.208184:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124de00. 00000400:00000200:1.0:1788971659.208190:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971659.208199:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971659.208203:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971659.208207:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179056896, portal 25 00000400:00000010:1.0:1788971659.208212:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588ee0. 00000100:00000200:1.0:1788971659.208214:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179056896, offset 0 00000400:00000200:1.0:1788971659.208219:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971659.208223:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971659.208227:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971659.208230:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971659.208232:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971659.208235:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971659.208238:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971659.208240:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [232:232:255:4294967295] 192.168.203.124@tcp LPNI seq info [232:232:7:4294967295] 00000400:00000200:1.0:1788971659.208244:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971659.208249:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971659.208253:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971659.208257:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71200. 00000800:00000200:1.0:1788971659.208261:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971659.208266:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971659.208268:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971659.208273:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971659.208276:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f180. 00000100:00000010:1.0:1788971659.208288:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f660. 00000800:00000200:2.0:1788971659.211153:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971659.211162:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55c00. 00000400:00000200:2.0:1788971659.211170:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971659.211178:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971659.211186:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480845d8 00000400:00000010:2.0:1788971659.211189:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480845d8. 00000100:00000200:2.0:1788971659.211201:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179057024/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971659.211284:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971659.211287:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71200. 00000400:00000200:2.0:1788971659.211290:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971659.211294:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971659.211297:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588ee0 00000400:00000010:2.0:1788971659.211333:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588ee0. 00000100:00000200:2.0:1788971659.211340:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179056896/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971675 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971659.214518:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971659.214528:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971659.214536:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971659.214540:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971659.214545:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971659.214550:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07100 00000400:00000200:3.0:1788971659.214555:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971659.214561:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd1d [1] + 224 00000400:00000200:3.0:1788971659.214575:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971659.214580:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971659.214593:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179056896/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971675 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971659.217126:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971659.217136:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971659.217144:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971659.217149:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971659.217154:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971659.217158:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07180 00000400:00000200:2.0:1788971659.217163:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971659.217169:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd15 [1] + 224 00000400:00000200:2.0:1788971659.217183:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971659.217201:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971659.217215:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179057024/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971675 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971659.217990:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971659.217997:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971659.217999:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971659.218007:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179057024/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971675 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971659.218017:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179057024/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971675 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971659.218057:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aae00. 02000000:00000010:0.0:1788971659.218064:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788971659.218068:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000400:00000010:1.0:1788971659.218329:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971659.218334:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971659.218336:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971659.218343:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179056896/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971675 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971659.218354:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179056896/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971675 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971659.218368:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124de00. 02000000:00000010:1.0:1788971659.218382:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55d00. 00000100:00000010:1.0:1788971659.218450:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000100:00000010:3.0:1788971659.911433:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f660. 00000100:00000010:2.0:1788971659.912454:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f180. 00000100:00000010:0.0:1788971664.327391:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788971664.327423:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55800. 00000100:00000010:0.0:1788971664.327474:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788971664.327479:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55300. 02000000:00000010:0.0:1788971664.327618:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aae00. 00000400:00000200:0.0:1788971664.327629:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971664.327633:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971664.327640:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971664.327648:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057280, portal 10 02000000:00000010:1.0:1788971664.327651:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c800. 00000400:00000200:1.0:1788971664.327657:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971664.327657:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084770. 00000400:00000010:1.0:1788971664.327660:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000100:00000200:0.0:1788971664.327660:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179057280, offset 0 00000400:00000010:1.0:1788971664.327664:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971664.327666:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057152, portal 25 00000400:00000010:1.0:1788971664.327669:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588e58. 00000400:00000200:0.0:1788971664.327670:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000100:00000200:1.0:1788971664.327673:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179057152, offset 0 00000400:00000200:1.0:1788971664.327677:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971664.327679:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971664.327687:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971664.327693:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971664.327697:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971664.327715:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971664.327718:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971664.327722:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [233:233:256:4294967295] 192.168.203.124@tcp LPNI seq info [233:233:8:4294967295] 00000400:00000200:0.0:1788971664.327726:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971664.327730:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971664.327733:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971664.327736:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971664.327737:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000400:00000200:1.0:1788971664.327739:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971664.327741:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971664.327743:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000800:00000200:0.0:1788971664.327744:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971664.327746:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [234:234:255:4294967295] 192.168.203.124@tcp LPNI seq info [234:234:7:4294967295] 00000400:00000200:1.0:1788971664.327749:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000010:0.0:1788971664.327749:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55400. 00000400:00000200:1.0:1788971664.327753:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971664.327756:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971664.327757:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971664.327762:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49d71200. 00000800:00000200:0.0:1788971664.327764:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971664.327766:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971664.327769:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971664.327769:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55400 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:1.0:1788971664.327771:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49d71200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971664.327790:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971664.327791:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971664.327796:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fcc0. 00000100:00000010:1.0:1788971664.327806:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f6c0. 00000800:00000200:3.0:1788971664.327937:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971664.327942:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55400. 00000400:00000200:3.0:1788971664.327949:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971664.327955:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971664.327960:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084770 00000400:00000010:3.0:1788971664.327962:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084770. 00000100:00000200:3.0:1788971664.327971:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179057280/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971664.328654:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971664.328660:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49d71200. 00000400:00000200:3.0:1788971664.328663:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971664.328669:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971664.328673:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588e58 00000400:00000010:3.0:1788971664.328675:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588e58. 00000100:00000200:3.0:1788971664.328681:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179057152/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971680 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971664.331164:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971664.331173:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971664.331181:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971664.331186:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971664.331191:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971664.331194:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07200 00000400:00000200:2.0:1788971664.331200:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971664.331206:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd2d [1] + 224 00000400:00000200:2.0:1788971664.331221:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971664.331226:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971664.331239:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179057152/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971680 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971664.336463:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971664.336475:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971664.336485:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971664.336490:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971664.336497:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971664.336502:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07280 00000400:00000200:3.0:1788971664.336520:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971664.336528:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd25 [1] + 224 00000400:00000200:3.0:1788971664.336554:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971664.336559:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971664.336580:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179057280/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971680 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971664.336837:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971664.336851:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971664.336854:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971664.336869:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179057280/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971680 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971664.336880:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179057280/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971680 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971664.336915:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aae00. 02000000:00000010:0.0:1788971664.336982:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55300. 00000100:00000010:0.0:1788971664.336996:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000400:00000010:1.0:1788971664.349213:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971664.349229:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971664.349233:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971664.349249:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179057152/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971680 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971664.349290:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179057152/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971680 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971664.349351:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c800. 02000000:00000010:1.0:1788971664.349359:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55800. 00000100:00000010:1.0:1788971664.349363:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000100:00000010:2.0:1788971665.031413:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fcc0. 00000100:00000010:3.0:1788971665.032328:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f6c0. 00000100:00000010:2.0:1788971669.451470:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788971669.451493:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d400. 00000100:00000010:2.0:1788971669.451523:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788971669.451527:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44c0d700. 02000000:00000010:2.0:1788971669.451562:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4e00. 00000400:00000200:2.0:1788971669.451572:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971669.451576:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788971669.451583:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971669.451589:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057408, portal 25 00000400:00000010:2.0:1788971669.451594:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788971669.451597:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179057408, offset 0 00000400:00000200:2.0:1788971669.451606:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971669.451614:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.451629:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971669.451635:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971669.451639:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971669.451642:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971669.451645:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971669.451649:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [235:235:256:4294967295] 192.168.203.124@tcp LPNI seq info [235:235:8:4294967295] 00000400:00000200:2.0:1788971669.451654:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.451663:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971669.451669:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971669.451675:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d300. 00000800:00000200:2.0:1788971669.451682:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971669.451689:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971669.451694:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971669.451707:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971669.451712:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7540. 02000000:00000010:3.0:1788971669.452225:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44549800. 00000400:00000200:3.0:1788971669.452232:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971669.452235:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e783c8. 00000400:00000010:3.0:1788971669.452240:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971669.452244:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057536, portal 10 00000400:00000010:3.0:1788971669.452249:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788971669.452251:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179057536, offset 0 00000400:00000200:3.0:1788971669.452257:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971669.452261:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971669.452265:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971669.452268:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971669.452271:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971669.452273:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971669.452275:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971669.452279:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [236:236:255:4294967295] 192.168.203.124@tcp LPNI seq info [236:236:7:4294967295] 00000400:00000200:3.0:1788971669.452282:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971669.452294:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000100:00000010:2.0:1788971669.452374:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7660. 00000800:00000200:2.0:1788971669.453831:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971669.453838:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d300. 00000400:00000200:2.0:1788971669.453844:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971669.453852:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971669.453859:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28 00000400:00000010:2.0:1788971669.453861:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788971669.453871:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179057408/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971669.455812:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971669.455824:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:3.0:1788971669.455831:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971669.455839:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971669.455843:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971669.455869:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971669.456065:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971669.456069:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:2.0:1788971669.456073:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971669.456078:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971669.456083:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:2.0:1788971669.456086:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:2.0:1788971669.456095:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179057536/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971685 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971669.460030:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460040:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971669.460049:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460053:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460058:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460062:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07300 00000400:00000200:2.0:1788971669.460067:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460073:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd35 [1] + 224 00000400:00000200:2.0:1788971669.460085:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971669.460089:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971669.460102:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179057408/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971685 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971669.460134:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460137:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971669.460141:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460143:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460146:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460149:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07380 00000400:00000200:2.0:1788971669.460152:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971669.460156:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd45 [1] + 224 00000400:00000200:2.0:1788971669.460163:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971669.460166:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971669.460171:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179057536/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971685 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971669.460204:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e783c8. 00000400:00000200:2.0:1788971669.460209:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:2.0:1788971669.460212:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:2.0:1788971669.460217:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179057536/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971685 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971669.460231:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179057536/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971685 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971669.460264:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44549800. 02000000:00000010:2.0:1788971669.460269:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d700. 00000100:00000010:2.0:1788971669.460274:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000400:00000010:2.0:1788971669.460291:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:2.0:1788971669.460294:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:2.0:1788971669.460296:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971669.460320:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179057408/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971685 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971669.460328:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179057408/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971685 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971669.460340:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4e00. 02000000:00000010:2.0:1788971669.460344:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44c0d400. 00000100:00000010:2.0:1788971669.460346:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000100:00000010:3.0:1788971670.152052:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7660. 00000100:00000010:2.0:1788971670.152358:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7540. 00000001:02000400:2.0:1788971670.528938:0:108442:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_280: @@@@@@ FAIL: Failed to unload modules before test execution Debug log: 3096 lines, 3096 kept, 0 dropped, 0 bad.