00000100:00000010:1.0F:1788971674.570323:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6680. 02000000:00000010:1.0:1788971674.570360:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6b00. 00000100:00000010:1.0:1788971674.570397:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971674.570402:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6400. 02000000:00000010:1.0:1788971674.570441:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4200. 00000400:00000200:1.0:1788971674.570451:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971674.570455:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971674.570461:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971674.570469:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057792, portal 10 00000400:00000010:1.0:1788971674.570475:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445884c8. 00000100:00000200:1.0:1788971674.570479:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179057792, offset 0 00000400:00000200:1.0:1788971674.570488:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971674.570496:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971674.570504:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971674.570509:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971674.570513:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971674.570517:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971674.570520:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971674.570524:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [237:237:256:4294967295] 192.168.203.124@tcp LPNI seq info [237:237:8:4294967295] 00000400:00000200:1.0:1788971674.570528:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971674.570538:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971674.570544:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971674.570549:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6c00. 00000800:00000200:1.0:1788971674.570556:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971674.570563:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971674.570567:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6c00 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:0.0F:1788971674.570685:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:0.0:1788971674.570698:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971674.570701:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971674.570705:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971674.570708:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057664, portal 25 00000400:00000010:0.0:1788971674.570711:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788971674.570713:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179057664, offset 0 00000400:00000200:0.0:1788971674.570718:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971674.570722:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971674.570725:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971674.570728:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971674.570730:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971674.570733:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971674.570736:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971674.570738:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [238:238:255:4294967295] 192.168.203.124@tcp LPNI seq info [238:238:7:4294967295] 00000400:00000200:0.0:1788971674.570742:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971674.570747:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971674.570751:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971674.570755:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800. 00000800:00000200:0.0:1788971674.570759:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971674.570763:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971674.570765:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971674.570789:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971674.570795:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9ff00. 00000100:00000010:1.0:1788971674.570805:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f180. 00000400:00000200:0.0:1788971674.573482:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0F:1788971674.577422:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971674.577443:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6c00. 00000400:00000200:2.0:1788971674.577452:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971674.577460:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971674.577467:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445884c8 00000400:00000010:2.0:1788971674.577470:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445884c8. 00000100:00000200:2.0:1788971674.577480:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179057792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971674.577548:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971674.577550:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800. 00000400:00000200:2.0:1788971674.577553:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971674.577557:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971674.577559:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:2.0:1788971674.577561:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:2.0:1788971674.577566:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6680 x1875872179057664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971690 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971674.586383:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.586394:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971674.586403:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.586407:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.586412:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.586416:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07480 00000400:00000200:2.0:1788971674.586421:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.586428:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd55 [1] + 224 00000400:00000200:2.0:1788971674.586440:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971674.586445:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971674.586457:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179057792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971690 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971674.587276:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971674.587283:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971674.587285:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971674.587345:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179057792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971690 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971674.587356:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179057792/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971690 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971674.587386:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4200. 02000000:00000010:1.0:1788971674.587392:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6400. 00000100:00000010:1.0:1788971674.587398:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 00000400:00000200:2.0:1788971674.601447:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.601456:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971674.601465:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.601482:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.601488:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.601492:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07400 00000400:00000200:2.0:1788971674.601497:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971674.601503:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd5d [1] + 224 00000400:00000200:2.0:1788971674.601516:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971674.601521:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971674.601533:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6680 x1875872179057664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971690 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971674.601639:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971674.601644:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971674.601648:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971674.601654:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6680 x1875872179057664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971690 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971674.601664:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6680 x1875872179057664/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971690 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971674.601689:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:0.0:1788971674.601700:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6b00. 00000100:00000010:0.0:1788971674.601706:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6680. 00000100:00000010:2.0:1788971675.271373:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9ff00. 00000100:00000010:3.0F:1788971675.271527:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f180. 00000100:00000010:3.0:1788971679.687968:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788971679.687999:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400. 00000100:00000010:3.0:1788971679.688039:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788971679.688043:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 02000000:00000010:3.0:1788971679.688084:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f1000. 00000400:00000200:3.0:1788971679.688096:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971679.688100:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78aa8. 00000400:00000010:3.0:1788971679.688108:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971679.688117:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179057920, portal 25 00000400:00000010:3.0:1788971679.688123:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971679.688126:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179057920, offset 0 00000400:00000200:3.0:1788971679.688137:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971679.688148:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.688156:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971679.688163:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971679.688167:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971679.688172:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971679.688175:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971679.688179:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [239:239:256:4294967295] 192.168.203.124@tcp LPNI seq info [239:239:8:4294967295] 00000400:00000200:3.0:1788971679.688183:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.688195:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971679.688201:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971679.688206:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:3.0:1788971679.688214:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971679.688223:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971679.688228:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971679.688243:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971679.688249:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859aa20. 00000100:00000010:3.0:1788971679.688262:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a540. 02000000:00000010:3.0:1788971679.688286:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0800. 00000400:00000200:3.0:1788971679.688289:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971679.688323:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78f20. 00000400:00000010:3.0:1788971679.688327:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081d48. 00000100:00000200:3.0:1788971679.688330:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058048, portal 10 00000400:00000010:3.0:1788971679.688336:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971679.688338:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179058048, offset 0 00000400:00000200:3.0:1788971679.688343:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971679.688346:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.688349:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971679.688352:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971679.688354:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971679.688357:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971679.688359:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971679.688362:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [240:240:255:4294967295] 192.168.203.124@tcp LPNI seq info [240:240:7:4294967295] 00000400:00000200:3.0:1788971679.688365:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.688370:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971679.688374:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971679.688378:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dc00. 00000800:00000200:3.0:1788971679.688382:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971679.688386:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971679.688389:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971679.688395:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971679.688453:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971679.688457:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:3.0:1788971679.688463:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971679.688468:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971679.688475:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:3.0:1788971679.688478:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971679.688489:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179057920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971679.688589:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971679.688592:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dc00. 00000400:00000200:3.0:1788971679.688601:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971679.688605:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971679.688608:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:3.0:1788971679.688610:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971679.688615:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179058048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971695 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971679.690181:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.690192:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971679.690200:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.690205:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.690211:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.690216:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07500 00000400:00000200:3.0:1788971679.690222:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.690230:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd65 [1] + 224 00000400:00000200:3.0:1788971679.690246:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971679.690251:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971679.690266:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179057920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971695 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971679.690343:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78aa8. 00000400:00000200:3.0:1788971679.690349:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:3.0:1788971679.690351:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971679.690359:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179057920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971695 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971679.690369:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179057920/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971695 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971679.690402:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f1000. 02000000:00000010:3.0:1788971679.690410:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400. 00000100:00000010:3.0:1788971679.690416:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000400:00000200:3.0:1788971679.691081:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.691087:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971679.691108:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.691112:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.691116:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.691119:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07580 00000400:00000200:3.0:1788971679.691124:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971679.691130:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd75 [1] + 224 00000400:00000200:3.0:1788971679.691152:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971679.691157:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971679.691167:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179058048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971695 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971679.691203:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78f20. 00000400:00000200:3.0:1788971679.691207:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081d48 00000400:00000010:3.0:1788971679.691209:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081d48. 00000100:00000200:3.0:1788971679.691215:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179058048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971695 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971679.691223:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179058048/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971695 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971679.691255:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0800. 02000000:00000010:3.0:1788971679.691259:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:3.0:1788971679.691262:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000100:00000010:3.0:1788971680.395829:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a540. 00000100:00000010:2.0:1788971680.397171:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859aa20. 00000001:02000400:1.0:1788971681.955053:0:109014:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 290: Check that lnet acceptor is using explicit address binding ========================================================== 12:34:40 (1788971680) 00000020:00000010:2.0:1788971682.298169:0:109041:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971682.298351:0:109041:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:1.0:1788971682.367033:0:109046:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971682.367138:0:109046:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:3.0:1788971683.402943:0:109051:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971683.403054:0:109051:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971684.458625:0:109055:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971684.459479:0:109055:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000100:00000010:2.0:1788971684.807969:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788971684.808001:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e300. 00000100:00000010:2.0:1788971684.808043:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788971684.808049:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e500. 02000000:00000010:3.0:1788971684.808248:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c200. 00000400:00000200:3.0:1788971684.808262:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971684.808267:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78f20. 00000400:00000010:3.0:1788971684.808275:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081d48. 00000100:00000200:3.0:1788971684.808284:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058176, portal 25 00000400:00000010:3.0:1788971684.808348:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971684.808353:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179058176, offset 0 00000400:00000200:3.0:1788971684.808364:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971684.808375:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.808385:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971684.808392:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971684.808397:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971684.808402:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971684.808405:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971684.808409:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [241:241:256:4294967295] 192.168.203.124@tcp LPNI seq info [241:241:8:4294967295] 00000400:00000200:3.0:1788971684.808414:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.808424:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971684.808432:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971684.808437:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788971684.808446:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971684.808454:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971684.808460:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971684.808476:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971684.808482:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859ac00. 00000100:00000010:3.0:1788971684.808496:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a540. 02000000:00000010:3.0:1788971684.808522:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c800. 00000400:00000200:3.0:1788971684.808526:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971684.808529:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78aa8. 00000400:00000010:3.0:1788971684.808533:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480814c8. 00000100:00000200:3.0:1788971684.808535:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058304, portal 10 00000400:00000010:3.0:1788971684.808542:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971684.808544:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179058304, offset 0 00000400:00000200:3.0:1788971684.808549:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971684.808552:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.808556:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971684.808559:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971684.808568:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971684.808570:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971684.808573:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971684.808575:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [242:242:255:4294967295] 192.168.203.124@tcp LPNI seq info [242:242:7:4294967295] 00000400:00000200:3.0:1788971684.808579:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.808585:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971684.808589:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971684.808593:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dc00. 00000800:00000200:3.0:1788971684.808597:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971684.808600:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971684.808602:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971684.808609:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971684.808689:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971684.808694:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788971684.808702:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971684.808708:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971684.808714:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:3.0:1788971684.808717:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971684.808729:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179058176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971684.808827:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971684.808830:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dc00. 00000400:00000200:3.0:1788971684.808832:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971684.808836:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971684.808840:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:3.0:1788971684.809260:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971684.809270:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179058304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971700 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971684.814682:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.814695:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971684.814705:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.814729:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.814734:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.814739:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07680 00000400:00000200:3.0:1788971684.814745:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.814752:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xd95 [1] + 224 00000400:00000200:3.0:1788971684.814770:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971684.814774:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971684.814793:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179058304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971700 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971684.815210:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78aa8. 00000400:00000200:2.0:1788971684.815219:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480814c8 00000400:00000010:2.0:1788971684.815222:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480814c8. 00000100:00000200:2.0:1788971684.815231:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179058304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971700 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971684.815242:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179058304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971700 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971684.815292:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c800. 02000000:00000010:2.0:1788971684.815341:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e500. 00000100:00000010:2.0:1788971684.815347:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000400:00000200:3.0:1788971684.815932:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.815938:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971684.815945:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.815949:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.815953:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.815958:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07600 00000400:00000200:3.0:1788971684.815962:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971684.815969:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xd85 [1] + 224 00000400:00000200:3.0:1788971684.815982:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971684.815986:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971684.815995:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179058176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971700 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971684.817486:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78f20. 00000400:00000200:2.0:1788971684.817504:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081d48 00000400:00000010:2.0:1788971684.817507:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081d48. 00000100:00000200:2.0:1788971684.817546:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179058176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971700 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971684.817558:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179058176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971700 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971684.817592:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c200. 02000000:00000010:2.0:1788971684.817601:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e300. 00000100:00000010:2.0:1788971684.817607:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000020:00000010:3.0:1788971685.502112:0:109059:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971685.502516:0:109059:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000100:00000010:2.0:1788971685.511435:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859ac00. 00000100:00000010:3.0:1788971685.511542:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a540. 00000020:00000010:3.0:1788971686.534676:0:109063:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971686.534762:0:109063:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:2.0:1788971687.563941:0:109067:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971687.564140:0:109067:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:0.0:1788971688.583772:0:109072:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971688.584005:0:109072:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:2.0:1788971689.621723:0:109076:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971689.622107:0:109076:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000100:00000010:0.0:1788971689.929110:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788971689.929131:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788971689.929167:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788971689.929171:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 02000000:00000010:1.0:1788971689.929344:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4a00. 00000400:00000200:1.0:1788971689.929354:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971689.929358:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971689.929365:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971689.929372:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058432, portal 25 00000400:00000010:1.0:1788971689.929377:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588d48. 00000100:00000200:1.0:1788971689.929381:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179058432, offset 0 00000400:00000200:1.0:1788971689.929390:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971689.929400:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971689.929408:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971689.929413:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971689.929417:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971689.929420:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971689.929423:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971689.929427:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [243:243:256:4294967295] 192.168.203.124@tcp LPNI seq info [243:243:8:4294967295] 00000400:00000200:1.0:1788971689.929431:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971689.929441:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971689.929447:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971689.929452:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6800. 00000800:00000200:1.0:1788971689.929474:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971689.929482:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971689.929486:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6800 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:0.0:1788971689.929611:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:0.0:1788971689.929618:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971689.929620:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971689.929624:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971689.929627:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058560, portal 10 00000400:00000010:0.0:1788971689.929632:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788971689.929634:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179058560, offset 0 00000400:00000200:0.0:1788971689.929638:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971689.929642:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971689.929646:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971689.929649:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971689.929651:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971689.929653:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971689.929655:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971689.929658:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [244:244:255:4294967295] 192.168.203.124@tcp LPNI seq info [244:244:7:4294967295] 00000400:00000200:0.0:1788971689.929661:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971689.929666:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971689.929670:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971689.929673:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500. 00000800:00000200:0.0:1788971689.929677:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971689.929681:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971689.929683:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971689.929705:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971689.929710:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fb40. 00000100:00000010:1.0:1788971689.929732:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f660. 00000400:00000200:0.0:1788971689.930129:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971689.930202:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971689.930206:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6800. 00000400:00000200:3.0:1788971689.930213:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971689.930218:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971689.930228:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588d48 00000400:00000010:3.0:1788971689.930231:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588d48. 00000100:00000200:3.0:1788971689.930241:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179058432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971689.934929:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971689.934939:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500. 00000400:00000200:3.0:1788971689.934945:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971689.934953:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971689.934959:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:3.0:1788971689.934962:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000400:00000200:2.0:1788971689.934962:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.934967:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000100:00000200:3.0:1788971689.934972:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179058560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971705 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971689.934973:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.934978:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.934983:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.934986:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07780 00000400:00000200:2.0:1788971689.934990:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.934995:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xdad [1] + 224 00000400:00000200:2.0:1788971689.935004:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971689.935008:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971689.935016:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179058560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971705 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971689.935159:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.935162:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971689.935171:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.935174:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.935177:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.935180:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07700 00000400:00000200:2.0:1788971689.935194:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971689.935199:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xda5 [1] + 224 00000400:00000200:2.0:1788971689.935209:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971689.935212:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971689.935221:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179058432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971705 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971689.935561:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971689.935566:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971689.935568:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971689.935575:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179058432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971705 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971689.935584:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179058432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971705 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971689.935619:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4a00. 02000000:00000010:1.0:1788971689.935625:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:1.0:1788971689.935629:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000400:00000010:0.0:1788971689.935746:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971689.935750:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971689.935752:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971689.935757:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179058560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971705 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971689.935765:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179058560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971705 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971689.935784:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:0.0:1788971689.935788:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788971689.935791:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000100:00000010:2.0:1788971690.632019:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fb40. 00000100:00000010:3.0:1788971690.632780:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f660. 00000020:00000010:0.0:1788971690.656064:0:109080:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971690.656146:0:109080:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971691.693755:0:109084:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971691.693853:0:109084:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:1.0:1788971692.729949:0:109088:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971692.730039:0:109088:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:2.0:1788971693.751997:0:109092:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971693.752100:0:109092:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971694.792715:0:109096:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971694.795097:0:109096:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000100:00000010:0.0:1788971695.047432:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788971695.047455:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788971695.047499:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788971695.047506:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:0.0:1788971695.047550:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3600. 00000400:00000200:0.0:1788971695.047559:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971695.047564:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971695.047570:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971695.047577:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058816, portal 10 00000400:00000010:0.0:1788971695.047583:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788971695.047586:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179058816, offset 0 00000400:00000200:0.0:1788971695.047595:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971695.047602:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971695.047610:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971695.047615:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971695.047620:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971695.047623:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971695.047639:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971695.047643:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [245:245:256:4294967295] 192.168.203.124@tcp LPNI seq info [245:245:8:4294967295] 00000400:00000200:0.0:1788971695.047647:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971695.047658:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971695.047663:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971695.047667:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55500. 00000800:00000200:0.0:1788971695.047674:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971695.047681:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971695.047685:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971695.047712:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971695.047718:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84780. 00000100:00000010:0.0:1788971695.047741:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e849c0. 02000000:00000010:0.0:1788971695.047761:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:0.0:1788971695.047766:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971695.047781:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163d68. 00000400:00000010:0.0:1788971695.047787:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788971695.047789:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058688, portal 25 00000400:00000010:0.0:1788971695.047792:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084770. 00000100:00000200:0.0:1788971695.047794:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179058688, offset 0 00000400:00000200:0.0:1788971695.047798:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971695.047802:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971695.047805:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971695.047808:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971695.047810:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971695.047812:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971695.047814:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971695.047817:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [246:246:255:4294967295] 192.168.203.124@tcp LPNI seq info [246:246:7:4294967295] 00000400:00000200:0.0:1788971695.047820:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971695.047825:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971695.047829:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971695.047839:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55d00. 00000800:00000200:0.0:1788971695.047843:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971695.047847:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971695.047849:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971695.047853:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971695.048075:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971695.048080:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55500. 00000400:00000200:3.0:1788971695.048087:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971695.048093:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971695.048097:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:3.0:1788971695.048109:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000100:00000200:3.0:1788971695.048129:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179058816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971695.048202:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971695.048205:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55d00. 00000400:00000200:3.0:1788971695.048207:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971695.048211:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971695.048213:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084770 00000400:00000010:3.0:1788971695.048215:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084770. 00000100:00000200:3.0:1788971695.048219:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179058688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971711 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971695.052635:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052644:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971695.052652:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052657:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052662:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052666:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07800 00000400:00000200:2.0:1788971695.052672:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052678:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xdbd [1] + 224 00000400:00000200:2.0:1788971695.052697:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971695.052703:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971695.052717:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179058688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971711 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971695.052947:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052951:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971695.052956:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052958:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052961:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052964:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07880 00000400:00000200:2.0:1788971695.052967:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971695.052986:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xdb5 [1] + 224 00000400:00000200:2.0:1788971695.053002:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971695.053004:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971695.053011:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179058816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971711 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971695.053110:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163d68. 00000400:00000200:0.0:1788971695.053115:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:0.0:1788971695.053117:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788971695.053125:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179058688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971711 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971695.053133:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179058688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971711 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971695.053155:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971695.053157:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971695.053159:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971695.053162:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179058816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971711 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971695.053169:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179058816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971711 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971695.053185:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:0.0:1788971695.053191:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788971695.053195:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788971695.053197:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3600. 02000000:00000010:0.0:1788971695.053200:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788971695.053202:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000100:00000010:3.0:1788971695.751429:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e849c0. 00000100:00000010:2.0:1788971695.751429:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84780. 00000020:00000010:2.0:1788971695.832535:0:109101:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971695.832625:0:109101:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971696.872174:0:109105:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971696.872645:0:109105:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:3.0:1788971697.893844:0:109109:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971697.895285:0:109109:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971698.926220:0:109113:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971698.926296:0:109113:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:0.0:1788971699.949265:0:109117:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971699.950786:0:109117:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000100:00000010:0.0:1788971700.167440:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788971700.167468:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788971700.167512:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788971700.167538:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 02000000:00000010:0.0:1788971700.167580:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3000. 00000400:00000200:0.0:1788971700.167600:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971700.167605:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971700.167612:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971700.167619:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059072, portal 10 00000400:00000010:0.0:1788971700.167626:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788971700.167635:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179059072, offset 0 00000400:00000200:0.0:1788971700.167643:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971700.167652:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971700.167660:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971700.167665:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971700.167669:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971700.167672:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971700.167675:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971700.167679:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [247:247:256:4294967295] 192.168.203.124@tcp LPNI seq info [247:247:8:4294967295] 00000400:00000200:0.0:1788971700.167683:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971700.167691:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971700.167697:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971700.167702:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800. 00000800:00000200:0.0:1788971700.167708:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971700.167715:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971700.167719:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971700.167748:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788971700.167752:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84000. 00000100:00000010:0.0:1788971700.167772:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84ea0. 02000000:00000010:0.0:1788971700.167790:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2800. 00000400:00000200:0.0:1788971700.167793:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971700.167796:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163d68. 00000400:00000010:0.0:1788971700.167799:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480845d8. 00000100:00000200:0.0:1788971700.167801:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179058944, portal 25 00000400:00000010:0.0:1788971700.167810:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084330. 00000100:00000200:0.0:1788971700.167812:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179058944, offset 0 00000400:00000200:0.0:1788971700.167816:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971700.167820:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971700.167823:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971700.167826:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971700.167828:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971700.167831:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971700.167833:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971700.167836:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [248:248:255:4294967295] 192.168.203.124@tcp LPNI seq info [248:248:7:4294967295] 00000400:00000200:0.0:1788971700.167839:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971700.167845:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971700.167850:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971700.167853:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55f00. 00000800:00000200:0.0:1788971700.167856:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971700.167859:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971700.167862:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971700.167865:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971700.168031:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971700.168036:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800. 00000400:00000200:3.0:1788971700.168043:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971700.168047:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971700.168056:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:3.0:1788971700.168058:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:3.0:1788971700.168069:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179059072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971700.168138:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971700.168141:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55f00. 00000400:00000200:3.0:1788971700.168144:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971700.168147:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971700.168149:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084330 00000400:00000010:3.0:1788971700.168151:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084330. 00000100:00000200:3.0:1788971700.168155:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179058944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971716 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971700.174287:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174296:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971700.174335:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174340:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174345:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174349:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07900 00000400:00000200:2.0:1788971700.174355:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174361:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xdcd [1] + 224 00000400:00000200:2.0:1788971700.174373:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971700.174378:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971700.174391:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179058944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971716 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971700.174430:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174434:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971700.174438:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174440:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174443:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174446:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07980 00000400:00000200:2.0:1788971700.174450:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971700.174464:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xdc5 [1] + 224 00000400:00000200:2.0:1788971700.174488:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971700.174492:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971700.174498:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179059072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971716 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971700.174952:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163d68. 00000400:00000200:0.0:1788971700.174958:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480845d8 00000400:00000010:0.0:1788971700.174961:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480845d8. 00000100:00000200:0.0:1788971700.175036:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179058944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971716 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971700.175055:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179058944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971716 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971700.175080:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971700.175083:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971700.175085:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971700.175089:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179059072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971716 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971700.175122:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179059072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971716 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971700.175154:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2800. 02000000:00000010:0.0:1788971700.175161:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788971700.175165:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788971700.175168:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3000. 02000000:00000010:0.0:1788971700.175171:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788971700.175173:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000100:00000010:2.0:1788971700.871395:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84000. 00000100:00000010:3.0:1788971700.871477:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84ea0. 00000020:00000010:3.0:1788971700.995066:0:109121:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971700.995163:0:109121:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:2.0:1788971702.039695:0:109125:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971702.039790:0:109125:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971703.071458:0:109130:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971703.071547:0:109130:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:1.0:1788971704.110329:0:109134:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971704.110422:0:109134:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:3.0:1788971705.165781:0:109138:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971705.165882:0:109138:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000100:00000010:2.0:1788971705.287658:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788971705.287689:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e300. 00000100:00000010:2.0:1788971705.287731:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788971705.287736:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e500. 02000000:00000010:3.0:1788971705.287883:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389de00. 00000400:00000200:3.0:1788971705.287894:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971705.287898:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78738. 00000400:00000010:3.0:1788971705.287905:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971705.287912:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059200, portal 25 00000400:00000010:3.0:1788971705.287918:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971705.287920:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179059200, offset 0 00000400:00000200:3.0:1788971705.287929:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971705.287961:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971705.287969:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971705.287986:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971705.287990:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971705.287993:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971705.287996:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971705.287999:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [249:249:256:4294967295] 192.168.203.124@tcp LPNI seq info [249:249:8:4294967295] 00000400:00000200:3.0:1788971705.288003:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971705.288012:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971705.288017:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971705.288022:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1dc00. 00000800:00000200:3.0:1788971705.288028:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971705.288035:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971705.288039:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1dc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971705.288098:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971705.288105:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a540. 00000100:00000010:3.0:1788971705.288114:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a8a0. 02000000:00000010:3.0:1788971705.288133:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c800. 00000400:00000200:3.0:1788971705.288138:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971705.288140:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78a50. 00000400:00000010:3.0:1788971705.288143:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788971705.288146:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059328, portal 10 00000400:00000010:3.0:1788971705.288150:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971705.288152:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179059328, offset 0 00000400:00000200:3.0:1788971705.288156:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971705.288160:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971705.288163:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971705.288165:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971705.288168:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971705.288170:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971705.288172:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971705.288180:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [250:250:255:4294967295] 192.168.203.124@tcp LPNI seq info [250:250:7:4294967295] 00000400:00000200:3.0:1788971705.288184:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971705.288190:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971705.288194:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971705.288197:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788971705.288201:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971705.288205:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971705.288207:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971705.288212:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971705.291173:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971705.291182:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1dc00. 00000400:00000200:3.0:1788971705.291190:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971705.291198:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971705.291205:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:3.0:1788971705.291207:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971705.291218:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179059200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971705.291349:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971705.291352:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788971705.291355:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971705.291359:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971705.291362:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788971705.291364:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971705.291369:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179059328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971721 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971705.299590:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299602:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971705.299612:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299617:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299623:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299627:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07a80 00000400:00000200:2.0:1788971705.299634:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299641:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xde5 [1] + 224 00000400:00000200:2.0:1788971705.299654:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971705.299658:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971705.299674:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179059328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971721 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971705.299725:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299729:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971705.299733:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299735:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299753:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299755:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07a00 00000400:00000200:2.0:1788971705.299759:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971705.299763:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xdd5 [1] + 224 00000400:00000200:2.0:1788971705.299773:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971705.299775:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971705.299782:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179059200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971721 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971705.300232:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78a50. 00000400:00000200:3.0:1788971705.300237:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:3.0:1788971705.300241:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788971705.300249:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179059328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971721 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971705.300283:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179059328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971721 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971705.300420:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78738. 00000400:00000200:3.0:1788971705.300424:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:3.0:1788971705.300426:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971705.300431:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179059200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971721 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971705.300438:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179059200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971721 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971705.300451:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c800. 02000000:00000010:3.0:1788971705.300457:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e500. 00000100:00000010:3.0:1788971705.300462:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 02000000:00000010:3.0:1788971705.300466:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389de00. 02000000:00000010:3.0:1788971705.300469:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e300. 00000100:00000010:3.0:1788971705.300471:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000100:00000010:2.0:1788971705.991437:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a540. 00000100:00000010:3.0:1788971705.991623:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a8a0. 00000020:00000010:3.0:1788971706.195688:0:109142:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971706.195811:0:109142:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971707.223248:0:109146:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971707.223492:0:109146:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971708.271419:0:109152:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971708.271510:0:109152:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:1.0:1788971709.305822:0:109156:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971709.305904:0:109156:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:0.0:1788971710.347140:0:109160:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971710.347415:0:109160:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000100:00000010:2.0:1788971710.407437:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410a80. 02000000:00000010:2.0:1788971710.407462:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e400. 00000100:00000010:2.0:1788971710.407503:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788971710.407508:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ed00. 02000000:00000010:3.0:1788971710.407620:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c800. 00000400:00000200:3.0:1788971710.407630:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971710.407635:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78738. 00000400:00000010:3.0:1788971710.407641:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971710.407648:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059456, portal 25 00000400:00000010:3.0:1788971710.407653:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081990. 00000100:00000200:3.0:1788971710.407656:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179059456, offset 0 00000400:00000200:3.0:1788971710.407687:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971710.407695:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971710.407776:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971710.407781:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971710.407785:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971710.407789:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971710.407792:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971710.407795:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [251:251:256:4294967295] 192.168.203.124@tcp LPNI seq info [251:251:8:4294967295] 00000400:00000200:3.0:1788971710.407800:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971710.407810:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971710.407817:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971710.407823:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:3.0:1788971710.407830:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971710.407837:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971710.407841:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971710.407862:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971710.407867:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a8a0. 00000100:00000010:3.0:1788971710.407878:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859ac60. 02000000:00000010:3.0:1788971710.407896:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389de00. 00000400:00000200:3.0:1788971710.407900:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971710.407902:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78a50. 00000400:00000010:3.0:1788971710.407906:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971710.407909:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059584, portal 10 00000400:00000010:3.0:1788971710.407913:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971710.407916:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179059584, offset 0 00000400:00000200:3.0:1788971710.407920:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971710.407924:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971710.407927:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971710.407929:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971710.407932:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971710.407934:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971710.407942:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971710.407945:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [252:252:255:4294967295] 192.168.203.124@tcp LPNI seq info [252:252:7:4294967295] 00000400:00000200:3.0:1788971710.407950:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971710.407956:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971710.408138:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971710.408142:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:2.0:1788971710.408147:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971710.408152:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971710.408157:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081990 00000400:00000010:2.0:1788971710.408159:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081990. 00000100:00000200:2.0:1788971710.408169:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410a80 x1875872179059456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971710.408396:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971710.408458:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e900. 00000800:00000200:2.0:1788971710.408465:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971710.408469:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971710.408473:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971710.408493:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971710.408550:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971710.408553:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e900. 00000400:00000200:3.0:1788971710.408556:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971710.408560:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971710.408564:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:3.0:1788971710.408566:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971710.408571:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179059584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971726 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971710.416049:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416059:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971710.416066:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416071:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416089:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416093:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07b00 00000400:00000200:2.0:1788971710.416097:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416104:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xdf5 [1] + 224 00000400:00000200:2.0:1788971710.416114:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971710.416119:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971710.416131:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410a80 x1875872179059456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971726 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971710.416176:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416180:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971710.416185:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416187:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416190:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416192:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07b80 00000400:00000200:2.0:1788971710.416196:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971710.416200:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe05 [1] + 224 00000400:00000200:2.0:1788971710.416208:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971710.416211:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971710.416217:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179059584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971726 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971710.416354:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78a50. 00000400:00000200:3.0:1788971710.416359:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788971710.416362:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971710.416369:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179059584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971726 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971710.416378:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179059584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971726 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971710.416413:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78738. 00000400:00000200:3.0:1788971710.416416:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:3.0:1788971710.416418:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971710.416422:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410a80 x1875872179059456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971726 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971710.416430:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410a80 x1875872179059456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971726 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971710.416444:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389de00. 02000000:00000010:3.0:1788971710.416613:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ed00. 00000100:00000010:3.0:1788971710.416626:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 02000000:00000010:3.0:1788971710.416631:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c800. 02000000:00000010:3.0:1788971710.416634:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e400. 00000100:00000010:3.0:1788971710.416636:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410a80. 00000100:00000010:2.0:1788971711.111430:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a8a0. 00000100:00000010:3.0:1788971711.111526:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859ac60. 00000020:00000010:0.0:1788971711.381918:0:109164:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971711.382010:0:109164:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:1.0:1788971712.426002:0:109168:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971712.426091:0:109168:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971713.478984:0:109172:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971713.479062:0:109172:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:0.0:1788971714.502188:0:109176:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971714.502275:0:109176:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000100:00000010:3.0:1788971715.533137:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788971715.533159:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dc00. 00000100:00000010:3.0:1788971715.533190:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788971715.533195:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d900. 02000000:00000010:2.0:1788971715.533247:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab400. 00000400:00000200:2.0:1788971715.533256:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971715.533260:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788971715.533267:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788971715.533275:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059712, portal 25 00000400:00000010:2.0:1788971715.533280:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788971715.533283:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179059712, offset 0 00000400:00000200:2.0:1788971715.533291:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971715.533327:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.533347:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971715.533352:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971715.533357:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971715.533360:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971715.533363:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971715.533367:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [253:253:256:4294967295] 192.168.203.124@tcp LPNI seq info [253:253:8:4294967295] 00000400:00000200:2.0:1788971715.533372:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.533381:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971715.533387:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971715.533391:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e600. 00000800:00000200:2.0:1788971715.533398:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971715.533406:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971715.533411:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971715.533423:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971715.533429:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0. 00000100:00000010:2.0:1788971715.533440:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f79c0. 02000000:00000010:2.0:1788971715.533458:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab800. 00000400:00000200:2.0:1788971715.533462:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971715.533464:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788971715.533468:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788971715.533470:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059840, portal 10 00000400:00000010:2.0:1788971715.533475:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8. 00000100:00000200:2.0:1788971715.533477:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179059840, offset 0 00000400:00000200:2.0:1788971715.533481:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971715.533485:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.533488:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971715.533490:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971715.533493:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971715.533495:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971715.533497:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971715.533500:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [254:254:255:4294967295] 192.168.203.124@tcp LPNI seq info [254:254:7:4294967295] 00000400:00000200:2.0:1788971715.533510:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.533517:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971715.533521:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971715.533524:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00. 00000800:00000200:2.0:1788971715.533528:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971715.533532:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971715.533534:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971715.533540:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971715.533639:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971715.533642:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e600. 00000400:00000200:3.0:1788971715.533649:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971715.533654:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971715.533659:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28 00000400:00000010:3.0:1788971715.533661:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28. 00000100:00000200:3.0:1788971715.533670:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179059712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971715.533746:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971715.533749:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00. 00000400:00000200:3.0:1788971715.533752:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971715.533755:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971715.533757:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8 00000400:00000010:3.0:1788971715.533759:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8. 00000100:00000200:3.0:1788971715.533764:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179059840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971731 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1788971715.536573:0:109181:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971715.536670:0:109181:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000400:00000200:2.0:1788971715.538077:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538085:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971715.538093:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538097:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538102:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538106:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07c80 00000400:00000200:2.0:1788971715.538111:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538117:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe25 [1] + 224 00000400:00000200:2.0:1788971715.538129:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971715.538134:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971715.538146:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179059840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971731 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971715.538427:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:3.0:1788971715.538433:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:3.0:1788971715.538436:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:3.0:1788971715.538443:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179059840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971731 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971715.538452:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179059840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971731 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971715.538495:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab800. 02000000:00000010:3.0:1788971715.538501:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d900. 00000100:00000010:3.0:1788971715.538506:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000400:00000200:2.0:1788971715.538561:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538565:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971715.538570:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538573:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538576:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538579:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07c00 00000400:00000200:2.0:1788971715.538584:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971715.538588:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xe15 [1] + 224 00000400:00000200:2.0:1788971715.538595:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971715.538604:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971715.538610:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179059712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971731 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971715.538779:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:3.0:1788971715.538783:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:3.0:1788971715.538785:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:3.0:1788971715.538790:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179059712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971731 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971715.538799:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179059712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971731 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971715.538811:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab400. 02000000:00000010:3.0:1788971715.538815:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dc00. 00000100:00000010:3.0:1788971715.538817:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000100:00000010:3.0:1788971716.231418:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f79c0. 00000100:00000010:2.0:1788971716.231418:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0. 00000020:00000010:2.0:1788971716.562176:0:109185:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971716.562261:0:109185:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971717.594561:0:109189:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971717.594726:0:109189:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:0.0:1788971718.638538:0:109193:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971718.638741:0:109193:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971719.674804:0:109197:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971719.674901:0:109197:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000100:00000010:0.0:1788971720.647825:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788971720.647846:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55a00. 00000100:00000010:0.0:1788971720.647883:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788971720.647888:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:0.0:1788971720.647919:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3e00. 00000400:00000200:0.0:1788971720.647927:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971720.647931:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 02000000:00000010:1.0:1788971720.647935:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5200. 00000400:00000010:0.0:1788971720.647938:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000400:00000200:1.0:1788971720.647944:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000200:0.0:1788971720.647947:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060096, portal 10 00000400:00000010:1.0:1788971720.647949:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:0.0:1788971720.647956:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480845d8. 00000400:00000010:1.0:1788971720.647957:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971720.647961:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179059968, portal 25 00000100:00000200:0.0:1788971720.647974:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179060096, offset 0 00000400:00000010:1.0:1788971720.647975:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588a18. 00000100:00000200:1.0:1788971720.647980:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179059968, offset 0 00000400:00000200:0.0:1788971720.647986:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971720.647996:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788971720.647997:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971720.648008:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971720.648012:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971720.648016:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971720.648020:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971720.648023:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971720.648027:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [255:255:256:4294967295] 192.168.203.124@tcp LPNI seq info [255:255:8:4294967295] 00000400:00000200:0.0:1788971720.648031:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971720.648036:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971720.648043:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971720.648047:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971720.648054:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971720.648058:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000400:00000200:1.0:1788971720.648061:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971720.648066:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000800:00000200:0.0:1788971720.648070:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971720.648073:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [256:256:255:4294967295] 192.168.203.124@tcp LPNI seq info [256:256:7:4294967295] 00000800:00000010:0.0:1788971720.648077:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55600. 00000400:00000200:1.0:1788971720.648081:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971720.648093:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788971720.648095:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971720.648101:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971720.648106:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400. 00000800:00000200:0.0:1788971720.648107:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971720.648114:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55600 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:1.0:1788971720.648115:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971720.648124:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971720.648127:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971720.648137:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971720.648142:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f180. 00000400:00000200:0.0:1788971720.648146:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971720.648158:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9ff00. 00000800:00000200:2.0:1788971720.648223:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971720.648228:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55600. 00000400:00000200:2.0:1788971720.648235:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971720.648240:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971720.648245:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480845d8 00000400:00000010:2.0:1788971720.648248:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480845d8. 00000100:00000200:2.0:1788971720.648256:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179060096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971720.648351:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971720.648354:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400. 00000400:00000200:2.0:1788971720.648357:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971720.648360:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971720.648363:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588a18 00000400:00000010:2.0:1788971720.648365:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588a18. 00000100:00000200:2.0:1788971720.648370:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179059968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971736 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971720.655795:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.655804:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971720.655813:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.655817:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.655822:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.655826:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07d00 00000400:00000200:2.0:1788971720.655831:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.655851:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xe3d [1] + 224 00000400:00000200:2.0:1788971720.655868:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971720.655872:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971720.655887:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179059968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971736 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971720.656324:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971720.656332:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971720.656335:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971720.656342:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179059968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971736 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971720.656354:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179059968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971736 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971720.656381:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5200. 02000000:00000010:1.0:1788971720.656387:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55a00. 00000100:00000010:1.0:1788971720.656391:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 00000400:00000200:2.0:1788971720.656571:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.656576:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971720.656581:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.656583:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.656599:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.656603:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07d80 00000400:00000200:2.0:1788971720.656607:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971720.656611:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe35 [1] + 224 00000400:00000200:2.0:1788971720.656622:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971720.656626:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971720.656633:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179060096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971736 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971720.656792:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971720.656797:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971720.656799:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971720.656805:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179060096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971736 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971720.656814:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179060096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971736 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971720.656831:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3e00. 02000000:00000010:0.0:1788971720.656836:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:0.0:1788971720.656838:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000020:00000010:3.0:1788971720.713127:0:109201:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971720.713194:0:109201:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000100:00000010:3.0:1788971721.351426:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9ff00. 00000100:00000010:2.0:1788971721.351520:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f180. 00000020:00000010:3.0:1788971721.742839:0:109205:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971721.742924:0:109205:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:2.0:1788971722.779851:0:109210:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971722.780094:0:109210:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:1.0:1788971723.825966:0:109214:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971723.826067:0:109214:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000020:00000010:2.0:1788971724.855477:0:109218:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971724.855582:0:109218:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000100:00000010:3.0:1788971725.771817:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788971725.771838:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dc00. 00000100:00000010:3.0:1788971725.771934:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788971725.771941:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d900. 02000000:00000010:3.0:1788971725.771971:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389de00. 00000400:00000200:3.0:1788971725.771981:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971725.771985:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78738. 00000400:00000010:3.0:1788971725.771991:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081a18. 00000100:00000200:3.0:1788971725.771997:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060352, portal 10 00000400:00000010:3.0:1788971725.772004:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971725.772007:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179060352, offset 0 00000400:00000200:3.0:1788971725.772028:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971725.772037:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.772043:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971725.772048:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971725.772052:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971725.772055:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971725.772058:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971725.772062:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [257:257:256:4294967295] 192.168.203.124@tcp LPNI seq info [257:257:8:4294967295] 00000400:00000200:3.0:1788971725.772065:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.772074:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971725.772080:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971725.772085:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d700. 00000800:00000200:3.0:1788971725.772091:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971725.772098:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971725.772102:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971725.772118:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971725.772122:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a420. 00000100:00000010:3.0:1788971725.772137:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859ac60. 02000000:00000010:3.0:1788971725.772156:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c800. 00000400:00000200:3.0:1788971725.772160:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971725.772163:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78a50. 00000400:00000010:3.0:1788971725.772166:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971725.772169:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060224, portal 25 00000400:00000010:3.0:1788971725.772172:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081b28. 00000100:00000200:3.0:1788971725.772174:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179060224, offset 0 00000400:00000200:3.0:1788971725.772178:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971725.772182:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.772185:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971725.772187:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971725.772190:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971725.772192:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971725.772200:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971725.772203:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [258:258:255:4294967295] 192.168.203.124@tcp LPNI seq info [258:258:7:4294967295] 00000400:00000200:3.0:1788971725.772207:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.772212:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971725.772217:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971725.772220:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:3.0:1788971725.772224:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971725.772227:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971725.772230:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971725.772234:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971725.772277:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971725.772281:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d700. 00000400:00000200:3.0:1788971725.772284:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971725.772289:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971725.772294:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788971725.772296:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971725.772325:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179060352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971725.772440:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971725.772443:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:3.0:1788971725.772446:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971725.772450:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971725.772453:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081b28 00000400:00000010:3.0:1788971725.772455:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081b28. 00000100:00000200:3.0:1788971725.772460:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179060224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971741 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971725.779981:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971725.779991:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971725.780000:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971725.780005:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971725.780010:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971725.780014:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07e00 00000400:00000200:2.0:1788971725.780018:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971725.780024:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xe55 [1] + 224 00000400:00000200:2.0:1788971725.780052:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971725.780057:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971725.780070:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179060224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971741 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971725.780333:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78a50. 00000400:00000200:3.0:1788971725.780350:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:3.0:1788971725.780352:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971725.780359:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179060224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971741 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971725.780381:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179060224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971741 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971725.780411:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c800. 02000000:00000010:3.0:1788971725.780418:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dc00. 00000100:00000010:3.0:1788971725.780423:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000400:00000200:3.0:1788971725.786122:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.786131:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971725.786140:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.786145:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.786149:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.786153:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07e80 00000400:00000200:3.0:1788971725.786158:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971725.786164:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe45 [1] + 224 00000400:00000200:3.0:1788971725.786178:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971725.786183:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971725.786197:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179060352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971741 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971725.786381:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78738. 00000400:00000200:2.0:1788971725.786387:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081a18 00000400:00000010:2.0:1788971725.786390:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081a18. 00000100:00000200:2.0:1788971725.786397:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179060352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971741 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971725.786407:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179060352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971741 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971725.786438:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389de00. 02000000:00000010:2.0:1788971725.786446:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d900. 00000100:00000010:2.0:1788971725.786451:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000020:00000010:1.0:1788971725.889524:0:109222:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308ef0. 00000020:00000010:1.0:1788971725.889632:0:109222:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308ef0. 00000100:00000010:3.0:1788971726.471416:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859ac60. 00000100:00000010:2.0:1788971726.471417:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a420. 00000020:00000010:2.0:1788971726.932351:0:109226:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971726.932449:0:109226:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:0.0:1788971727.978460:0:109230:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971727.979147:0:109230:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971729.020509:0:109234:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971729.020631:0:109234:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:3.0:1788971730.068859:0:109239:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971730.068953:0:109239:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000100:00000010:3.0:1788971730.887945:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788971730.887966:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dc00. 00000100:00000010:3.0:1788971730.887991:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788971730.887995:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200. 02000000:00000010:2.0:1788971730.888094:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa600. 00000400:00000200:2.0:1788971730.888104:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971730.888109:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971730.888115:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971730.888122:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060480, portal 25 00000400:00000010:2.0:1788971730.888128:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788971730.888131:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179060480, offset 0 00000400:00000200:2.0:1788971730.888139:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971730.888147:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971730.888153:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971730.888158:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971730.888162:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971730.888166:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971730.888182:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971730.888186:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [259:259:256:4294967295] 192.168.203.124@tcp LPNI seq info [259:259:8:4294967295] 00000400:00000200:2.0:1788971730.888191:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971730.888201:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971730.888207:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971730.888212:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00. 00000800:00000200:2.0:1788971730.888218:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971730.888226:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971730.888230:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971730.888252:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971730.888257:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0. 00000100:00000010:2.0:1788971730.888268:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7c00. 02000000:00000010:2.0:1788971730.888287:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab800. 00000400:00000200:2.0:1788971730.888291:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971730.888294:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971730.888298:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0880. 00000100:00000200:2.0:1788971730.888504:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060608, portal 10 00000400:00000010:2.0:1788971730.888526:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b07f8. 00000100:00000200:2.0:1788971730.888529:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179060608, offset 0 00000400:00000200:2.0:1788971730.888533:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971730.888538:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971730.888542:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971730.888544:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971730.888547:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971730.888550:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971730.888552:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971730.888555:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [260:260:255:4294967295] 192.168.203.124@tcp LPNI seq info [260:260:7:4294967295] 00000400:00000200:2.0:1788971730.888559:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971730.888565:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971730.888569:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971730.888578:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e600. 00000800:00000200:2.0:1788971730.888583:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971730.888587:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971730.888589:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971730.888595:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971730.889638:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971730.889644:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00. 00000400:00000200:3.0:1788971730.889652:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971730.889659:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971730.889665:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:3.0:1788971730.889676:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:3.0:1788971730.889686:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179060480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971730.889760:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971730.889763:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e600. 00000400:00000200:3.0:1788971730.889766:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971730.889770:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971730.889773:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b07f8 00000400:00000010:3.0:1788971730.889774:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b07f8. 00000100:00000200:3.0:1788971730.889779:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179060608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971746 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971730.891975:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.891983:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971730.891991:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.891996:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892000:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892004:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e07f80 00000400:00000200:3.0:1788971730.892009:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892015:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe75 [1] + 224 00000400:00000200:3.0:1788971730.892028:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971730.892033:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971730.892048:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179060608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971746 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971730.892165:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971730.892171:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0880 00000400:00000010:2.0:1788971730.892173:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0880. 00000100:00000200:2.0:1788971730.892180:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179060608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971746 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971730.892189:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179060608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971746 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971730.892221:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab800. 02000000:00000010:2.0:1788971730.892228:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200. 00000100:00000010:2.0:1788971730.892233:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000400:00000200:3.0:1788971730.892778:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892784:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971730.892790:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892794:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892797:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892800:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e07f00 00000400:00000200:3.0:1788971730.892803:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971730.892822:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xe65 [1] + 224 00000400:00000200:3.0:1788971730.892998:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971730.893003:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971730.893010:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179060480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971746 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971730.896007:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788971730.896018:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8 00000400:00000010:2.0:1788971730.896021:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971730.896033:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179060480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971746 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971730.896045:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179060480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971746 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971730.896074:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa600. 02000000:00000010:2.0:1788971730.896080:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dc00. 00000100:00000010:2.0:1788971730.896085:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000020:00000010:1.0:1788971731.119103:0:109243:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971731.119187:0:109243:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308f70. 00000100:00000010:3.0:1788971731.591616:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7c00. 00000100:00000010:2.0:1788971731.592787:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0. 00000020:00000010:3.0:1788971732.175171:0:109247:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971732.175256:0:109247:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:2.0:1788971733.217783:0:109251:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971733.218328:0:109251:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000020:00000010:3.0:1788971734.268408:0:109255:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971734.268504:0:109255:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:1.0:1788971735.298664:0:109259:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971735.298754:0:109259:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308f70. 00000100:00000010:0.0:1788971736.007812:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f100. 02000000:00000010:0.0:1788971736.007835:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55b00. 02000000:00000010:1.0:1788971736.008069:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4c00. 00000400:00000200:1.0:1788971736.008080:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971736.008084:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971736.008091:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971736.008097:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060736, portal 25 00000400:00000010:1.0:1788971736.008103:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588990. 00000100:00000200:1.0:1788971736.008106:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179060736, offset 0 00000400:00000200:1.0:1788971736.008116:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971736.008124:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971736.008132:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971736.008137:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971736.008141:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971736.008144:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971736.008148:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971736.008152:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [261:261:256:4294967295] 192.168.203.124@tcp LPNI seq info [261:261:8:4294967295] 00000400:00000200:1.0:1788971736.008157:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971736.008166:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971736.008173:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971736.008178:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400. 00000800:00000200:1.0:1788971736.008185:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971736.008193:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971736.008197:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971736.008219:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971736.008237:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fba0. 00000100:00000010:1.0:1788971736.008248:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f840. 00000800:00000200:3.0:1788971736.008408:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971736.008413:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400. 00000400:00000200:3.0:1788971736.008419:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971736.008425:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971736.008431:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588990 00000400:00000010:3.0:1788971736.008433:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588990. 00000100:00000200:3.0:1788971736.008441:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f100 x1875872179060736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788971736.008971:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c700. 02000000:00000010:0.0:1788971736.008981:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55000. 02000000:00000010:1.0:1788971736.009805:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4400. 00000400:00000200:1.0:1788971736.009815:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971736.009819:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971736.009825:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588b28. 00000100:00000200:1.0:1788971736.009830:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060864, portal 10 00000400:00000010:1.0:1788971736.009837:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588f68. 00000100:00000200:1.0:1788971736.009840:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179060864, offset 0 00000400:00000200:1.0:1788971736.009847:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971736.009854:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971736.009861:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971736.009865:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971736.009869:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971736.009872:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971736.009874:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971736.009877:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [262:262:256:4294967295] 192.168.203.124@tcp LPNI seq info [262:262:8:4294967295] 00000400:00000200:1.0:1788971736.009882:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971736.009891:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971736.009896:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971736.009900:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6800. 00000800:00000200:1.0:1788971736.009906:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971736.009913:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971736.009917:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971736.009955:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971736.010657:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971736.010662:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6800. 00000400:00000200:3.0:1788971736.010667:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971736.010673:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971736.010679:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588f68 00000400:00000010:3.0:1788971736.010681:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588f68. 00000100:00000200:3.0:1788971736.010689:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c700 x1875872179060864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971752 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971736.014505:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014514:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971736.014523:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014528:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014534:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014551:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08000 00000400:00000200:3.0:1788971736.014557:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014564:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xe85 [1] + 224 00000400:00000200:3.0:1788971736.014576:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971736.014581:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971736.014594:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f100 x1875872179060736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971752 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971736.014633:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014637:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971736.014642:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014645:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014648:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014651:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08080 00000400:00000200:3.0:1788971736.014655:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971736.014659:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe8d [1] + 224 00000400:00000200:3.0:1788971736.014668:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971736.014672:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971736.014678:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c700 x1875872179060864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971752 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971736.014787:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971736.014793:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588b28 00000400:00000010:1.0:1788971736.014795:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588b28. 00000100:00000200:1.0:1788971736.014802:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c700 x1875872179060864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971752 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971736.014812:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c700 x1875872179060864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971752 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971736.014858:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971736.014862:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971736.014864:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971736.014869:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f100 x1875872179060736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971752 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971736.014878:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f100 x1875872179060736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971752 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971736.014892:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4400. 02000000:00000010:1.0:1788971736.014898:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55000. 00000100:00000010:1.0:1788971736.014904:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c700. 02000000:00000010:1.0:1788971736.014908:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4c00. 02000000:00000010:1.0:1788971736.014910:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55b00. 00000100:00000010:1.0:1788971736.014912:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f100. 00000020:00000010:3.0:1788971736.347691:0:109263:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971736.347985:0:109263:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000100:00000010:3.0:1788971736.711409:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f840. 00000100:00000010:2.0:1788971736.711409:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fba0. 00000020:00000010:1.0:1788971737.386724:0:109268:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971737.386805:0:109268:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308f70. 00000020:00000010:0.0:1788971738.420124:0:109272:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971738.420220:0:109272:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971739.458961:0:109276:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971739.459047:0:109276:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000020:00000010:1.0:1788971740.489277:0:109280:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971740.489404:0:109280:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308f70. 00000100:00000010:0.0:1788971741.128759:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788971741.128781:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55900. 00000100:00000010:0.0:1788971741.128824:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788971741.128829:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55c00. 02000000:00000010:1.0:1788971741.129052:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5c00. 00000400:00000200:1.0:1788971741.129062:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971741.129066:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971741.129073:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971741.129080:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179060992, portal 25 00000400:00000010:1.0:1788971741.129101:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588b28. 00000100:00000200:1.0:1788971741.129105:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179060992, offset 0 00000400:00000200:1.0:1788971741.129114:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971741.129122:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971741.129130:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971741.129135:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971741.129139:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971741.129143:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971741.129157:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971741.129185:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [263:263:256:4294967295] 192.168.203.124@tcp LPNI seq info [263:263:8:4294967295] 00000400:00000200:1.0:1788971741.129191:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971741.129201:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971741.129229:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971741.129234:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6b00. 00000800:00000200:1.0:1788971741.129254:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971741.129270:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971741.129275:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971741.129365:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971741.129371:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f6c0. 00000100:00000010:1.0:1788971741.129416:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f960. 02000000:00000010:1.0:1788971741.129460:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4400. 00000400:00000200:1.0:1788971741.129466:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971741.129469:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000800:00000200:3.0:1788971741.129470:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000010:1.0:1788971741.129473:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000800:00000010:3.0:1788971741.129476:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6b00. 00000100:00000200:1.0:1788971741.129478:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061120, portal 10 00000400:00000200:3.0:1788971741.129484:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1788971741.129485:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788971741.129490:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179061120, offset 0 00000400:00000200:3.0:1788971741.129493:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:1.0:1788971741.129499:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971741.129503:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.129504:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588b28 00000400:00000200:1.0:1788971741.129508:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000010:3.0:1788971741.129509:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588b28. 00000400:00000200:1.0:1788971741.129516:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971741.129522:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000100:00000200:3.0:1788971741.129523:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179060992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788971741.129526:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971741.129532:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971741.129538:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [264:264:255:4294967295] 192.168.203.124@tcp LPNI seq info [264:264:7:4294967295] 00000400:00000200:1.0:1788971741.129542:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971741.129554:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971741.129560:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971741.129571:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6c00. 00000800:00000200:1.0:1788971741.129586:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971741.129591:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971741.129594:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971741.129607:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971741.129704:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971741.129707:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6c00. 00000400:00000200:3.0:1788971741.129711:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971741.129715:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971741.129718:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:3.0:1788971741.129720:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:3.0:1788971741.129725:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179061120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971757 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971741.131561:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.131569:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971741.131577:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.131581:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.131586:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.131589:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08180 00000400:00000200:3.0:1788971741.131594:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.131600:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xe9d [1] + 224 00000400:00000200:3.0:1788971741.131612:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971741.131617:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971741.131630:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179061120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971757 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971741.137126:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.137136:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971741.137145:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.137150:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.137155:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.137163:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08100 00000400:00000200:3.0:1788971741.137168:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971741.137175:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xe95 [1] + 224 00000400:00000200:3.0:1788971741.137188:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971741.137206:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971741.137219:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179060992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971757 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971741.137885:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971741.137892:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:1.0:1788971741.137895:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788971741.137903:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179061120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971757 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971741.137913:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179061120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971757 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971741.137945:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971741.137947:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971741.137950:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971741.137954:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179060992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971757 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971741.137963:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179060992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971757 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971741.137978:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4400. 02000000:00000010:1.0:1788971741.137984:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55c00. 00000100:00000010:1.0:1788971741.137989:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 02000000:00000010:1.0:1788971741.137993:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5c00. 02000000:00000010:1.0:1788971741.137995:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55900. 00000100:00000010:1.0:1788971741.137998:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000020:00000010:2.0:1788971741.521405:0:109284:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb46166060. 00000020:00000010:2.0:1788971741.521502:0:109284:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb46166060. 00000100:00000010:2.0:1788971741.831970:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f6c0. 00000100:00000010:3.0:1788971741.836487:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f960. 00000020:00000010:1.0:1788971742.566102:0:109288:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971742.566186:0:109288:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971743.598699:0:109292:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb4a308f70. 00000020:00000010:1.0:1788971743.598786:0:109292:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb4a308f70. 00000020:00000010:3.0:1788971744.637633:0:109297:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb44ccaa00. 00000020:00000010:3.0:1788971744.637717:0:109297:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb44ccaa00. 00000020:00000010:0.0:1788971744.665957:0:109298:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff98bb45d94690. 00000020:00000010:0.0:1788971744.666058:0:109298:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff98bb45d94690. 00000100:00000010:3.0:1788971746.247789:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788971746.247810:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 00000100:00000010:3.0:1788971746.247847:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788971746.247854:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dc00. 02000000:00000010:2.0:1788971746.252123:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa600. 00000400:00000200:2.0:1788971746.252137:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971746.252142:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971746.252148:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971746.252155:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061248, portal 25 00000400:00000010:2.0:1788971746.252161:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0880. 00000100:00000200:2.0:1788971746.252164:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179061248, offset 0 00000400:00000200:2.0:1788971746.252172:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971746.252181:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971746.252188:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971746.252193:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971746.252197:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971746.252200:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971746.252203:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971746.252207:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [265:265:256:4294967295] 192.168.203.124@tcp LPNI seq info [265:265:8:4294967295] 00000400:00000200:2.0:1788971746.252212:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971746.252221:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971746.252227:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971746.252232:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e400. 00000800:00000200:2.0:1788971746.252239:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971746.252246:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971746.252251:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971746.252275:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971746.252281:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f70c0. 00000100:00000010:2.0:1788971746.252292:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0. 02000000:00000010:2.0:1788971746.252353:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab400. 00000400:00000200:2.0:1788971746.252358:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971746.252361:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971746.252364:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788971746.252380:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061376, portal 10 00000400:00000010:2.0:1788971746.252385:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0aa0. 00000100:00000200:2.0:1788971746.252387:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179061376, offset 0 00000400:00000200:2.0:1788971746.252392:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971746.252396:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971746.252400:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971746.252402:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971746.252405:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971746.252417:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971746.252420:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971746.252423:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [266:266:255:4294967295] 192.168.203.124@tcp LPNI seq info [266:266:7:4294967295] 00000400:00000200:2.0:1788971746.252427:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971746.252432:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971746.252436:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971746.252440:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ed00. 00000800:00000200:2.0:1788971746.252444:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971746.252448:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971746.252450:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ed00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971746.252456:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971746.252852:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971746.252857:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e400. 00000400:00000200:3.0:1788971746.252863:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971746.252868:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971746.252873:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0880 00000400:00000010:3.0:1788971746.252876:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0880. 00000100:00000200:3.0:1788971746.252885:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179061248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971746.252962:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971746.252965:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ed00. 00000400:00000200:3.0:1788971746.252968:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971746.252971:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971746.252974:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0aa0 00000400:00000010:3.0:1788971746.252976:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0aa0. 00000100:00000200:3.0:1788971746.252981:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6680 x1875872179061376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971762 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971746.255504:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.255514:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971746.255522:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.255527:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.255531:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.255548:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08280 00000400:00000200:3.0:1788971746.255553:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.255559:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xeb5 [1] + 224 00000400:00000200:3.0:1788971746.255573:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971746.255576:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971746.255591:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6680 x1875872179061376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971762 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971746.255888:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971746.255893:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:2.0:1788971746.255895:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788971746.255902:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6680 x1875872179061376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971762 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971746.255910:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6680 x1875872179061376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971762 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971746.255954:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab400. 02000000:00000010:2.0:1788971746.255960:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dc00. 00000100:00000010:2.0:1788971746.255966:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6680. 00000400:00000200:3.0:1788971746.257404:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.257421:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971746.257428:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.257432:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.257437:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.257441:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08200 00000400:00000200:3.0:1788971746.257446:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971746.257452:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xea5 [1] + 224 00000400:00000200:3.0:1788971746.257476:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971746.257481:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971746.257496:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179061248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971762 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971746.257581:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788971746.257595:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8 00000400:00000010:2.0:1788971746.257598:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971746.257604:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179061248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971762 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971746.257613:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179061248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971762 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971746.257634:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa600. 02000000:00000010:2.0:1788971746.257639:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:2.0:1788971746.257643:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000100:00000010:2.0:1788971746.952928:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f70c0. 00000100:00000010:3.0:1788971746.953035:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0. 00000100:00000010:1.0:1788971751.367475:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971751.367510:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971751.367554:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7100. 02000000:00000010:1.0:1788971751.367560:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6b00. 02000000:00000010:1.0:1788971751.367592:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:1.0:1788971751.367601:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971751.367606:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971751.367612:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971751.367618:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061632, portal 10 00000400:00000010:1.0:1788971751.367625:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788971751.367629:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179061632, offset 0 00000400:00000200:1.0:1788971751.367638:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971751.367646:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971751.367654:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971751.367660:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971751.367664:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971751.367667:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971751.367670:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971751.367674:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [267:267:256:4294967295] 192.168.203.124@tcp LPNI seq info [267:267:8:4294967295] 00000400:00000200:1.0:1788971751.367678:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971751.367687:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971751.367694:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971751.367698:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6800. 00000800:00000200:1.0:1788971751.367705:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971751.367713:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971751.367717:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971751.367738:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971751.367743:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fc60. 00000100:00000010:1.0:1788971751.367764:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f540. 02000000:00000010:1.0:1788971751.367786:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:1.0:1788971751.367790:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971751.367793:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971751.367802:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788971751.367805:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061504, portal 25 00000400:00000010:1.0:1788971751.367808:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588c38. 00000100:00000200:1.0:1788971751.367810:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179061504, offset 0 00000400:00000200:1.0:1788971751.367814:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971751.367818:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971751.367822:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971751.367824:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971751.367827:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971751.367829:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971751.367831:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971751.367834:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [268:268:255:4294967295] 192.168.203.124@tcp LPNI seq info [268:268:7:4294967295] 00000400:00000200:1.0:1788971751.367837:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971751.367842:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971751.367846:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971751.367849:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400. 00000800:00000200:1.0:1788971751.367853:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971751.367857:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971751.367859:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971751.367863:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971751.368487:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971751.368492:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6800. 00000400:00000200:3.0:1788971751.368499:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971751.368505:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971751.368511:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:3.0:1788971751.368513:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:3.0:1788971751.368522:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7100 x1875872179061632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971751.368601:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971751.368618:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400. 00000400:00000200:3.0:1788971751.368622:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971751.368626:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971751.368630:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588c38 00000400:00000010:3.0:1788971751.368631:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588c38. 00000100:00000200:3.0:1788971751.368636:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179061504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971767 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971751.372806:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372816:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971751.372824:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372829:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372834:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372838:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08300 00000400:00000200:3.0:1788971751.372843:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372849:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xecd [1] + 224 00000400:00000200:3.0:1788971751.372861:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971751.372866:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971751.372880:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179061504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971767 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971751.372919:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372923:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971751.372927:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372929:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372932:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372935:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08380 00000400:00000200:3.0:1788971751.372938:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971751.372944:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xec5 [1] + 224 00000400:00000200:3.0:1788971751.372952:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971751.372956:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971751.372962:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7100 x1875872179061632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971767 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971751.373584:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971751.373590:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445882a8 00000400:00000010:1.0:1788971751.373592:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788971751.373600:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179061504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971767 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971751.373609:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179061504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971767 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971751.373633:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971751.373635:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971751.373637:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971751.373642:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7100 x1875872179061632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971767 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971751.373657:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7100 x1875872179061632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971767 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971751.373678:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:1.0:1788971751.373684:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971751.373688:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971751.373692:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:1.0:1788971751.373694:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6b00. 00000100:00000010:1.0:1788971751.373696:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7100. 00000100:00000010:2.0:1788971752.071425:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fc60. 00000100:00000010:3.0:1788971752.071427:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f540. 00000100:00000010:1.0:1788971756.487436:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7100. 02000000:00000010:1.0:1788971756.487458:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6b00. 00000100:00000010:1.0:1788971756.487515:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971756.487521:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6c00. 02000000:00000010:1.0:1788971756.487555:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:1.0:1788971756.487564:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971756.487568:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971756.487574:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971756.487582:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061888, portal 10 00000400:00000010:1.0:1788971756.487589:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788971756.487592:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179061888, offset 0 00000400:00000200:1.0:1788971756.487601:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971756.487609:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971756.487630:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971756.487635:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971756.487639:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971756.487643:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971756.487646:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971756.487650:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [269:269:256:4294967295] 192.168.203.124@tcp LPNI seq info [269:269:8:4294967295] 00000400:00000200:1.0:1788971756.487655:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971756.487664:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971756.487690:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971756.487699:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400. 00000800:00000200:1.0:1788971756.487707:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 02000000:00000010:0.0:1788971756.487710:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50f41800. 00000400:00000200:0.0:1788971756.487716:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788971756.487718:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163580. 00000400:00000010:0.0:1788971756.487722:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971756.487724:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179061760, portal 25 00000400:00000010:0.0:1788971756.487727:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788971756.487728:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179061760, offset 0 00000400:00000200:0.0:1788971756.487733:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000800:00000200:1.0:1788971756.487737:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000400:00000200:0.0:1788971756.487740:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:1.0:1788971756.487745:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971756.487746:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788971756.487751:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971756.487754:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788971756.487756:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971756.487758:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788971756.487760:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [270:270:255:4294967295] 192.168.203.124@tcp LPNI seq info [270:270:7:4294967295] 00000400:00000200:0.0:1788971756.487762:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788971756.487768:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000400:00000200:1.0:1788971756.487772:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971756.487780:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f960. 00000800:00000200:0.0:1788971756.487781:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788971756.487786:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55a00. 00000800:00000200:0.0:1788971756.487790:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788971756.487793:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788971756.487795:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55a00 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:1.0:1788971756.487798:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f6c0. 00000400:00000200:0.0:1788971756.487801:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971756.487964:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971756.487969:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400. 00000400:00000200:3.0:1788971756.487976:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971756.487981:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971756.487986:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445882a8 00000400:00000010:3.0:1788971756.487988:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445882a8. 00000100:00000200:3.0:1788971756.487997:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179061888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971756.488080:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971756.488083:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55a00. 00000400:00000200:3.0:1788971756.488086:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971756.488089:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971756.488092:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:3.0:1788971756.488094:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:3.0:1788971756.488099:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7100 x1875872179061760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971772 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971756.491283:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491292:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971756.491356:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491361:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491365:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491370:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08480 00000400:00000200:3.0:1788971756.491375:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491381:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xed5 [1] + 224 00000400:00000200:3.0:1788971756.491393:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971756.491397:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971756.491411:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179061888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971772 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971756.491466:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491470:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971756.491474:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491477:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491479:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491482:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08400 00000400:00000200:3.0:1788971756.491486:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971756.491491:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xedd [1] + 224 00000400:00000200:3.0:1788971756.491509:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971756.491513:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971756.491520:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7100 x1875872179061760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971772 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971756.491577:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163580. 00000400:00000200:0.0:1788971756.491584:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:0.0:1788971756.491586:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788971756.491593:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7100 x1875872179061760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971772 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971756.491603:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7100 x1875872179061760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971772 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971756.491627:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50f41800. 02000000:00000010:0.0:1788971756.491633:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6b00. 00000100:00000010:0.0:1788971756.491637:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7100. 00000400:00000010:1.0:1788971756.491704:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971756.491709:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971756.491711:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971756.491719:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179061888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971772 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971756.491728:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179061888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971772 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971756.491746:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:1.0:1788971756.491751:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971756.491754:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 00000100:00000010:3.0:1788971757.191440:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f6c0. 00000100:00000010:2.0:1788971757.191443:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f960. 00000100:00000010:3.0:1788971761.607640:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7100. 02000000:00000010:3.0:1788971761.607790:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200. 00000100:00000010:3.0:1788971761.607830:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5c00. 02000000:00000010:3.0:1788971761.607836:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d900. 02000000:00000010:2.0:1788971761.607961:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa400. 00000400:00000200:2.0:1788971761.607972:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971761.607976:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971761.607983:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971761.607990:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062016, portal 25 00000400:00000010:2.0:1788971761.607995:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788971761.607997:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179062016, offset 0 00000400:00000200:2.0:1788971761.608006:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971761.608014:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971761.608022:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971761.608027:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971761.608031:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971761.608035:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971761.608037:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971761.608041:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [271:271:256:4294967295] 192.168.203.124@tcp LPNI seq info [271:271:8:4294967295] 00000400:00000200:2.0:1788971761.608045:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971761.608052:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971761.608059:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971761.608064:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e900. 00000800:00000200:2.0:1788971761.608084:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971761.608091:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971761.608095:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971761.608205:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971761.608211:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f74e0. 00000100:00000010:2.0:1788971761.608221:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7240. 02000000:00000010:2.0:1788971761.608240:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000. 00000400:00000200:2.0:1788971761.608244:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971761.608247:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971761.608250:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0110. 00000100:00000200:2.0:1788971761.608253:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062144, portal 10 00000400:00000010:2.0:1788971761.608259:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0770. 00000100:00000200:2.0:1788971761.608261:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179062144, offset 0 00000400:00000200:2.0:1788971761.608266:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971761.608269:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971761.608273:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971761.608276:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971761.608278:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971761.608279:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971761.608282:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971761.608284:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [272:272:255:4294967295] 192.168.203.124@tcp LPNI seq info [272:272:7:4294967295] 00000400:00000200:2.0:1788971761.608288:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971761.608294:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971761.608347:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971761.608351:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e300. 00000800:00000200:2.0:1788971761.608354:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971761.608358:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971761.608361:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971761.608366:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971761.608486:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971761.608490:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e900. 00000400:00000200:3.0:1788971761.608497:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971761.608503:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971761.608508:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:3.0:1788971761.608510:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:3.0:1788971761.608520:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7100 x1875872179062016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971761.610216:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971761.610223:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e300. 00000400:00000200:3.0:1788971761.610230:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971761.610237:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971761.610257:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0770 00000400:00000010:3.0:1788971761.610260:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0770. 00000100:00000200:3.0:1788971761.610269:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5c00 x1875872179062144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971777 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971761.613262:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613269:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971761.613275:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613278:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613281:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613284:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08580 00000400:00000200:3.0:1788971761.613288:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613293:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xef5 [1] + 224 00000400:00000200:3.0:1788971761.613413:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971761.613418:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971761.613432:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5c00 x1875872179062144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971777 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971761.613522:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613526:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971761.613540:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613543:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613546:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613549:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08500 00000400:00000200:3.0:1788971761.613553:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971761.613558:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xee5 [1] + 224 00000400:00000200:3.0:1788971761.613569:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971761.613573:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971761.613581:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7100 x1875872179062016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971777 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971761.613826:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971761.613831:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0110 00000400:00000010:2.0:1788971761.613847:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0110. 00000100:00000200:2.0:1788971761.613914:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5c00 x1875872179062144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971777 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971761.613923:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5c00 x1875872179062144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971777 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971761.613951:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788971761.613954:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8 00000400:00000010:2.0:1788971761.613956:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971761.613960:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7100 x1875872179062016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971777 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971761.613967:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7100 x1875872179062016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971777 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971761.613980:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000. 02000000:00000010:2.0:1788971761.613984:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d900. 00000100:00000010:2.0:1788971761.613989:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5c00. 02000000:00000010:2.0:1788971761.613992:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa400. 02000000:00000010:2.0:1788971761.613994:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200. 00000100:00000010:2.0:1788971761.613996:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7100. 00000100:00000010:2.0:1788971762.311562:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f74e0. 00000100:00000010:3.0:1788971762.311825:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7240. 00000100:00000010:2.0:1788971766.728370:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788971766.728392:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e500. 00000100:00000010:2.0:1788971766.728433:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411c00. 02000000:00000010:2.0:1788971766.728439:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e100. 02000000:00000010:2.0:1788971766.728471:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000. 00000400:00000200:2.0:1788971766.728482:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971766.728486:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971766.728493:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971766.728499:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062400, portal 10 00000400:00000010:2.0:1788971766.728506:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0110. 00000100:00000200:2.0:1788971766.728509:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179062400, offset 0 00000400:00000200:2.0:1788971766.728529:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971766.728537:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971766.728544:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971766.728550:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971766.728553:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971766.728557:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971766.728560:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971766.728564:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [273:273:256:4294967295] 192.168.203.124@tcp LPNI seq info [273:273:8:4294967295] 00000400:00000200:2.0:1788971766.728568:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971766.728577:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971766.728583:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971766.728588:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ef00. 00000800:00000200:2.0:1788971766.728594:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971766.728601:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971766.728605:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ef00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971766.728618:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971766.728622:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f74e0. 00000100:00000010:2.0:1788971766.728631:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7840. 02000000:00000010:2.0:1788971766.728656:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aaa00. 00000400:00000200:2.0:1788971766.728660:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971766.728662:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971766.728666:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788971766.728668:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062272, portal 25 00000400:00000010:2.0:1788971766.728672:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330. 00000100:00000200:2.0:1788971766.728674:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179062272, offset 0 00000400:00000200:2.0:1788971766.728678:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971766.728681:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971766.728684:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971766.728687:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971766.728689:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971766.728691:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971766.728703:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971766.728706:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [274:274:255:4294967295] 192.168.203.124@tcp LPNI seq info [274:274:7:4294967295] 00000400:00000200:2.0:1788971766.728710:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971766.728716:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971766.728721:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971766.728724:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e200. 00000800:00000200:2.0:1788971766.728727:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971766.728732:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971766.728734:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971766.728738:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971766.728839:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971766.728843:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ef00. 00000400:00000200:3.0:1788971766.728849:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971766.728864:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971766.728870:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0110 00000400:00000010:3.0:1788971766.728873:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0110. 00000100:00000200:3.0:1788971766.728883:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411c00 x1875872179062400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971766.728946:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971766.728949:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e200. 00000400:00000200:3.0:1788971766.728952:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971766.728955:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971766.728958:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330 00000400:00000010:3.0:1788971766.728959:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330. 00000100:00000200:3.0:1788971766.728964:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43412d80 x1875872179062272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971782 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971766.730658:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.730666:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971766.730674:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.730678:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.730682:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.730686:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08680 00000400:00000200:3.0:1788971766.730690:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.730696:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf05 [1] + 224 00000400:00000200:3.0:1788971766.730710:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971766.730713:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971766.730725:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411c00 x1875872179062400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971782 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971766.731329:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788971766.731336:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8 00000400:00000010:2.0:1788971766.731338:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971766.731346:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411c00 x1875872179062400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971782 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971766.731356:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411c00 x1875872179062400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971782 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971766.731386:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000. 02000000:00000010:2.0:1788971766.731392:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e100. 00000100:00000010:2.0:1788971766.731396:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411c00. 00000400:00000200:3.0:1788971766.743993:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.744006:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971766.744018:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.744043:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.744049:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.744065:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08600 00000400:00000200:3.0:1788971766.744071:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971766.744078:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xf15 [1] + 224 00000400:00000200:3.0:1788971766.744095:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971766.744101:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971766.744119:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43412d80 x1875872179062272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971782 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971766.745752:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971766.745764:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660 00000400:00000010:2.0:1788971766.745767:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788971766.745782:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43412d80 x1875872179062272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971782 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971766.745795:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43412d80 x1875872179062272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971782 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971766.745829:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aaa00. 02000000:00000010:2.0:1788971766.745836:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e500. 00000100:00000010:2.0:1788971766.745862:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412d80. 00000100:00000010:3.0:1788971767.431412:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7840. 00000100:00000010:2.0:1788971767.433436:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f74e0. 00000100:00000010:1.0:1788971771.847528:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971771.847559:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971771.847620:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80. 02000000:00000010:1.0:1788971771.847627:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6800. 02000000:00000010:1.0:1788971771.847672:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:1.0:1788971771.847685:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971771.847689:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971771.847698:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971771.847708:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062656, portal 10 00000400:00000010:1.0:1788971771.847716:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445886e8. 00000100:00000200:1.0:1788971771.847720:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179062656, offset 0 00000400:00000200:1.0:1788971771.847732:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971771.847760:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971771.847769:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971771.847775:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971771.847780:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971771.847784:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971771.847787:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971771.847791:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [275:275:256:4294967295] 192.168.203.124@tcp LPNI seq info [275:275:8:4294967295] 00000400:00000200:1.0:1788971771.847796:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971771.847808:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971771.847815:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971771.847823:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6b00. 00000800:00000200:1.0:1788971771.847831:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971771.847841:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971771.847846:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971771.847875:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971771.847882:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9fba0. 00000100:00000010:1.0:1788971771.847910:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f840. 02000000:00000010:1.0:1788971771.847937:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:1.0:1788971771.847942:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971771.847945:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971771.847949:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788971771.847952:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062528, portal 25 00000400:00000010:1.0:1788971771.847956:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588aa0. 00000100:00000200:1.0:1788971771.847958:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179062528, offset 0 00000400:00000200:1.0:1788971771.847962:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971771.847966:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971771.847970:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971771.847973:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971771.847976:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971771.847978:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971771.847980:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971771.847989:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [276:276:255:4294967295] 192.168.203.124@tcp LPNI seq info [276:276:7:4294967295] 00000400:00000200:1.0:1788971771.847993:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971771.847999:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971771.848003:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971771.848007:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400. 00000800:00000200:1.0:1788971771.848011:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971771.848015:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971771.848017:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971771.848022:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971771.848211:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971771.848216:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6b00. 00000400:00000200:3.0:1788971771.848235:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971771.848241:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971771.848248:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445886e8 00000400:00000010:3.0:1788971771.848250:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445886e8. 00000100:00000200:3.0:1788971771.848263:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6d80 x1875872179062656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971771.848376:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971771.848379:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400. 00000400:00000200:3.0:1788971771.848383:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971771.848386:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971771.848389:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588aa0 00000400:00000010:3.0:1788971771.848391:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588aa0. 00000100:00000200:3.0:1788971771.848395:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179062528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971787 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971771.854912:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971771.854925:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971771.854936:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971771.854942:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971771.854948:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971771.854953:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08780 00000400:00000200:2.0:1788971771.854959:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971771.854966:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf25 [1] + 224 00000400:00000200:2.0:1788971771.854985:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971771.854990:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971771.855008:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6d80 x1875872179062656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971787 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971771.857873:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971771.857888:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788971771.857892:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971771.857919:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6d80 x1875872179062656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971787 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971771.857932:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6d80 x1875872179062656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971787 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971771.857977:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:1.0:1788971771.857986:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6800. 00000100:00000010:1.0:1788971771.857991:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80. 00000400:00000200:3.0:1788971771.863695:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971771.863709:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971771.863719:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971771.863725:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971771.863731:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971771.863755:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08700 00000400:00000200:3.0:1788971771.863762:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971771.863770:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xf2d [1] + 224 00000400:00000200:3.0:1788971771.863787:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971771.863792:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971771.863812:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179062528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971787 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971771.864131:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971771.864150:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330 00000400:00000010:1.0:1788971771.864153:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788971771.864162:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179062528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971787 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971771.864173:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179062528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971787 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971771.864207:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:1.0:1788971771.864215:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971771.864221:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 00000100:00000010:2.0:1788971772.552511:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9fba0. 00000100:00000010:3.0:1788971772.553085:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f840. 00000100:00000010:3.0:1788971776.967894:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4700. 02000000:00000010:3.0:1788971776.967916:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788971776.967953:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7100. 02000000:00000010:3.0:1788971776.967959:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d400. 02000000:00000010:3.0:1788971776.968008:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016c00. 02000000:00000010:2.0:1788971776.968009:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aaa00. 00000400:00000200:3.0:1788971776.968021:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971776.968021:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971776.968028:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:3.0:1788971776.968029:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78a50. 00000400:00000010:3.0:1788971776.968038:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000400:00000010:2.0:1788971776.968038:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660. 00000100:00000200:3.0:1788971776.968046:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062784, portal 25 00000100:00000200:2.0:1788971776.968047:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179062912, portal 10 00000400:00000010:3.0:1788971776.968055:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081b28. 00000400:00000010:2.0:1788971776.968057:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8. 00000100:00000200:3.0:1788971776.968061:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179062784, offset 0 00000100:00000200:2.0:1788971776.968062:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179062912, offset 0 00000400:00000200:2.0:1788971776.968073:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971776.968081:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971776.968084:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971776.968093:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971776.968098:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971776.968102:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971776.968105:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971776.968108:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971776.968111:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [277:277:256:4294967295] 192.168.203.124@tcp LPNI seq info [277:277:8:4294967295] 00000400:00000200:2.0:1788971776.968116:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971776.968139:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971776.968147:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971776.968149:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000010:2.0:1788971776.968153:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e500. 00000400:00000200:3.0:1788971776.968158:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000800:00000200:2.0:1788971776.968163:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000400:00000200:3.0:1788971776.968164:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971776.968170:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971776.968176:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000800:00000200:2.0:1788971776.968177:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000400:00000200:3.0:1788971776.968181:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000800:00000200:2.0:1788971776.968184:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971776.968199:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [278:278:255:4294967295] 192.168.203.124@tcp LPNI seq info [278:278:7:4294967295] 00000400:00000200:3.0:1788971776.968206:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.968214:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000400:00000200:2.0:1788971776.968216:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971776.968224:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000100:00000010:2.0:1788971776.968224:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f74e0. 00000100:00000010:2.0:1788971776.968235:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7ea0. 00000800:00000010:3.0:1788971776.968238:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:3.0:1788971776.968245:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971776.968251:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971776.968255:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971776.968261:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971776.968290:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971776.968295:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e500. 00000400:00000200:2.0:1788971776.968339:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971776.968345:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971776.968350:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8 00000400:00000010:2.0:1788971776.968353:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971776.968361:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7100 x1875872179062912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971776.968440:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971776.968443:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:2.0:1788971776.968447:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971776.968450:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971776.968453:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081b28 00000400:00000010:2.0:1788971776.968455:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081b28. 00000100:00000200:2.0:1788971776.968460:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4700 x1875872179062784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971792 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971776.981714:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981725:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971776.981735:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981739:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981744:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981749:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08880 00000400:00000200:3.0:1788971776.981754:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981760:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf3d [1] + 224 00000400:00000200:3.0:1788971776.981771:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971776.981776:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971776.981790:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7100 x1875872179062912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971792 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971776.981829:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981833:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971776.981837:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981840:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981843:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981846:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08800 00000400:00000200:3.0:1788971776.981893:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971776.981898:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xf35 [1] + 224 00000400:00000200:3.0:1788971776.981907:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971776.981910:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971776.981917:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4700 x1875872179062784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971792 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971776.981957:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78a50. 00000400:00000200:3.0:1788971776.981974:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:3.0:1788971776.981976:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971776.981982:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4700 x1875872179062784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971792 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971776.981992:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4700 x1875872179062784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971792 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971776.982017:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016c00. 02000000:00000010:3.0:1788971776.982023:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788971776.982027:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4700. 00000400:00000010:2.0:1788971776.982262:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971776.982268:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660 00000400:00000010:2.0:1788971776.982270:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788971776.982277:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7100 x1875872179062912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971792 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971776.982288:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7100 x1875872179062912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971792 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971776.982340:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aaa00. 02000000:00000010:2.0:1788971776.982344:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d400. 00000100:00000010:2.0:1788971776.982348:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7100. 00000100:00000010:2.0:1788971777.671439:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f74e0. 00000100:00000010:3.0:1788971777.673129:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7ea0. 00000100:00000010:3.0:1788971782.087425:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4700. 02000000:00000010:3.0:1788971782.087449:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 00000100:00000010:3.0:1788971782.087491:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5c00. 02000000:00000010:3.0:1788971782.087496:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 02000000:00000010:2.0:1788971782.087845:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aaa00. 00000400:00000200:2.0:1788971782.087856:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971782.087860:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971782.087867:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788971782.087874:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063040, portal 25 00000400:00000010:2.0:1788971782.087879:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b05d8. 00000100:00000200:2.0:1788971782.087882:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179063040, offset 0 00000400:00000200:2.0:1788971782.087890:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971782.087898:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971782.087905:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971782.087910:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971782.087914:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971782.087918:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971782.087921:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971782.087925:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [279:279:256:4294967295] 192.168.203.124@tcp LPNI seq info [279:279:8:4294967295] 00000400:00000200:2.0:1788971782.087929:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971782.087938:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971782.087944:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971782.087949:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e500. 00000800:00000200:2.0:1788971782.087956:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971782.087963:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971782.087968:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971782.087992:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971782.087998:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f74e0. 00000100:00000010:2.0:1788971782.088008:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f72a0. 02000000:00000010:2.0:1788971782.088045:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000. 00000400:00000200:2.0:1788971782.088050:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971782.088053:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971782.088056:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0088. 00000100:00000200:2.0:1788971782.088059:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063168, portal 10 00000400:00000010:2.0:1788971782.088064:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0550. 00000100:00000200:2.0:1788971782.088065:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179063168, offset 0 00000400:00000200:2.0:1788971782.088069:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971782.088073:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971782.088076:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971782.088079:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971782.088081:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971782.088084:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971782.088086:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971782.088088:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [280:280:255:4294967295] 192.168.203.124@tcp LPNI seq info [280:280:7:4294967295] 00000400:00000200:2.0:1788971782.088092:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971782.088098:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971782.088102:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971782.088105:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e100. 00000800:00000200:2.0:1788971782.088109:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971782.088114:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971782.088117:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971782.088122:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971782.088229:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971782.088233:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e500. 00000400:00000200:3.0:1788971782.088240:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971782.088245:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971782.088250:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b05d8 00000400:00000010:3.0:1788971782.088252:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b05d8. 00000100:00000200:3.0:1788971782.088262:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4700 x1875872179063040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971782.088357:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971782.088360:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e100. 00000400:00000200:3.0:1788971782.088363:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971782.088367:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971782.088370:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0550 00000400:00000010:3.0:1788971782.088371:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0550. 00000100:00000200:3.0:1788971782.088376:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5c00 x1875872179063168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971798 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971782.092362:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.092381:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971782.092388:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.092392:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.092397:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.092401:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08980 00000400:00000200:3.0:1788971782.092406:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.092411:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf65 [1] + 224 00000400:00000200:3.0:1788971782.092423:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971782.092427:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971782.092439:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5c00 x1875872179063168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971798 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971782.094228:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.094235:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971782.094241:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.094245:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.094249:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.094253:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08900 00000400:00000200:3.0:1788971782.094258:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971782.094263:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xf55 [1] + 224 00000400:00000200:3.0:1788971782.094273:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971782.094277:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971782.094288:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4700 x1875872179063040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971798 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971782.096654:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788971782.096664:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0088 00000400:00000010:2.0:1788971782.096667:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0088. 00000100:00000200:2.0:1788971782.096679:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5c00 x1875872179063168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971798 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971782.096827:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5c00 x1875872179063168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971798 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971782.096879:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788971782.096882:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660 00000400:00000010:2.0:1788971782.096883:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788971782.096888:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4700 x1875872179063040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971798 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971782.096896:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4700 x1875872179063040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971798 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971782.096911:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000. 02000000:00000010:2.0:1788971782.096916:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:2.0:1788971782.096922:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5c00. 02000000:00000010:2.0:1788971782.096926:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aaa00. 02000000:00000010:2.0:1788971782.096928:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:2.0:1788971782.096930:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4700. 00000100:00000010:3.0:1788971782.791415:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f72a0. 00000100:00000010:2.0:1788971782.791416:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f74e0. 00000100:00000010:1.0:1788971787.207524:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971787.207546:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971787.207589:0:37060:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb6d80. 02000000:00000010:1.0:1788971787.207594:0:37060:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb516f6800. 02000000:00000010:1.0:1788971787.207628:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:1.0:1788971787.207638:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971787.207642:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971787.207648:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788971787.207655:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063424, portal 10 00000400:00000010:1.0:1788971787.207663:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788971787.207666:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179063424, offset 0 00000400:00000200:1.0:1788971787.207673:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971787.207682:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971787.207690:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971787.207695:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971787.207700:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971787.207703:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971787.207706:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971787.207710:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [281:281:256:4294967295] 192.168.203.124@tcp LPNI seq info [281:281:8:4294967295] 00000400:00000200:1.0:1788971787.207714:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971787.207725:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971787.207731:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971787.207737:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6400. 00000800:00000200:1.0:1788971787.207744:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971787.207750:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971787.207755:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971787.207778:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971787.207784:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f480. 00000100:00000010:1.0:1788971787.207805:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f180. 02000000:00000010:1.0:1788971787.207828:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:1.0:1788971787.207833:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971787.207849:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971787.207854:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788971787.207857:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063296, portal 25 00000400:00000010:1.0:1788971787.207860:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588110. 00000100:00000200:1.0:1788971787.207863:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179063296, offset 0 00000400:00000200:1.0:1788971787.207868:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971787.207871:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971787.207875:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971787.207878:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971787.207881:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971787.207883:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971787.207885:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971787.207888:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [282:282:255:4294967295] 192.168.203.124@tcp LPNI seq info [282:282:7:4294967295] 00000400:00000200:1.0:1788971787.207892:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971787.207898:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971787.207903:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971787.207906:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6b00. 00000800:00000200:1.0:1788971787.207910:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971787.207915:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971787.207918:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971787.207922:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971787.208100:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971787.208105:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6400. 00000400:00000200:3.0:1788971787.208111:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971787.208125:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971787.208130:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:3.0:1788971787.208132:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:3.0:1788971787.208148:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb6d80 x1875872179063424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971787.208549:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971787.208553:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6b00. 00000400:00000200:3.0:1788971787.208556:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971787.208560:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971787.208564:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588110 00000400:00000010:3.0:1788971787.208565:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588110. 00000100:00000200:3.0:1788971787.208571:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb7800 x1875872179063296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971803 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971787.211178:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.211187:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971787.211195:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.211200:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.211205:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.211209:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08a00 00000400:00000200:3.0:1788971787.212745:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.212755:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xf7d [1] + 224 00000400:00000200:3.0:1788971787.212774:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971787.212779:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971787.212794:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb7800 x1875872179063296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971803 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971787.212839:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.212843:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971787.212849:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.212853:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.212857:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.212859:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08a80 00000400:00000200:3.0:1788971787.212863:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971787.212867:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf75 [1] + 224 00000400:00000200:3.0:1788971787.212890:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971787.212893:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971787.212900:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb6d80 x1875872179063424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971803 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971787.213360:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971787.213367:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588660 00000400:00000010:1.0:1788971787.213369:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788971787.213376:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb7800 x1875872179063296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971803 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971787.213386:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb7800 x1875872179063296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971803 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971787.213408:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971787.213411:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330 00000400:00000010:1.0:1788971787.213412:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788971787.213426:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb6d80 x1875872179063424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971803 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971787.213434:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb6d80 x1875872179063424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971803 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971787.213454:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:1.0:1788971787.213461:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6c00. 00000100:00000010:1.0:1788971787.213466:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb7800. 02000000:00000010:1.0:1788971787.213470:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:1.0:1788971787.213472:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb516f6800. 00000100:00000010:1.0:1788971787.213485:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb6d80. 00000100:00000010:3.0:1788971787.911423:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f180. 00000100:00000010:2.0:1788971787.911590:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f480. 00000100:00000010:0.0:1788971792.327490:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788971792.327512:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55600. 00000100:00000010:0.0:1788971792.327552:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788971792.327557:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55f00. 02000000:00000010:1.0:1788971792.327642:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2200. 00000400:00000200:1.0:1788971792.327652:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971792.327656:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1f78. 00000400:00000010:1.0:1788971792.327663:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788971792.327670:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063552, portal 25 00000400:00000010:1.0:1788971792.327675:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588660. 00000100:00000200:1.0:1788971792.327679:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179063552, offset 0 00000400:00000200:1.0:1788971792.327687:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971792.327696:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971792.327703:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971792.327719:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971792.327724:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971792.327727:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971792.327730:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971792.327734:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [283:283:256:4294967295] 192.168.203.124@tcp LPNI seq info [283:283:8:4294967295] 00000400:00000200:1.0:1788971792.327738:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971792.327748:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971792.327754:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971792.327759:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6800. 00000800:00000200:1.0:1788971792.327765:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971792.327785:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971792.327789:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971792.327811:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788971792.327816:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9ff00. 00000100:00000010:1.0:1788971792.327826:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51a9f660. 02000000:00000010:1.0:1788971792.327850:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2600. 00000400:00000200:1.0:1788971792.327855:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788971792.327858:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1688. 00000400:00000010:1.0:1788971792.327861:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788971792.327864:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063680, portal 10 00000400:00000010:1.0:1788971792.327869:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588550. 00000100:00000200:1.0:1788971792.327872:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179063680, offset 0 00000400:00000200:1.0:1788971792.327876:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788971792.327880:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971792.327883:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788971792.327886:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971792.327889:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788971792.327891:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971792.327893:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788971792.327895:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [284:284:255:4294967295] 192.168.203.124@tcp LPNI seq info [284:284:7:4294967295] 00000400:00000200:1.0:1788971792.327899:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788971792.327904:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788971792.327908:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788971792.327912:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb516f6c00. 00000800:00000200:1.0:1788971792.327916:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788971792.327920:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788971792.327923:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb516f6c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971792.327928:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971792.328104:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971792.328110:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6800. 00000400:00000200:3.0:1788971792.328126:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971792.328131:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971792.328136:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588660 00000400:00000010:3.0:1788971792.328138:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588660. 00000100:00000200:3.0:1788971792.328148:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179063552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971792.328237:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971792.328240:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb516f6c00. 00000400:00000200:3.0:1788971792.328243:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971792.328247:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971792.328249:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588550 00000400:00000010:3.0:1788971792.328251:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588550. 00000100:00000200:3.0:1788971792.328256:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179063680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971808 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971792.330600:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.330609:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971792.330616:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.330620:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.330625:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.330629:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08b00 00000400:00000200:3.0:1788971792.330634:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.330639:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xf85 [1] + 224 00000400:00000200:3.0:1788971792.330652:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971792.330668:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971792.330680:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179063552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971808 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971792.330956:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1f78. 00000400:00000200:1.0:1788971792.330962:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330 00000400:00000010:1.0:1788971792.330964:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788971792.330975:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179063552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971808 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971792.330984:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179063552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971808 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971792.331008:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2200. 02000000:00000010:1.0:1788971792.331014:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55600. 00000100:00000010:1.0:1788971792.331019:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000400:00000200:3.0:1788971792.331164:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.331169:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971792.331173:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.331175:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.331189:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.331193:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08b80 00000400:00000200:3.0:1788971792.331196:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971792.331201:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf8d [1] + 224 00000400:00000200:3.0:1788971792.331209:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971792.331212:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971792.331219:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179063680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971808 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971792.331375:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1688. 00000400:00000200:1.0:1788971792.331379:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588770 00000400:00000010:1.0:1788971792.331381:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588770. 00000100:00000200:1.0:1788971792.331386:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179063680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971808 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971792.331395:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179063680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971808 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971792.331412:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2600. 02000000:00000010:1.0:1788971792.331415:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55f00. 00000100:00000010:1.0:1788971792.331417:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000100:00000010:3.0:1788971793.031435:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9f660. 00000100:00000010:2.0:1788971793.031849:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51a9ff00. 00000100:00000010:3.0:1788971797.447958:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4380. 02000000:00000010:3.0:1788971797.447979:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1da00. 00000100:00000010:3.0:1788971797.448030:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788971797.448035:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d300. 02000000:00000010:3.0:1788971797.448079:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017400. 00000400:00000200:3.0:1788971797.448097:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971797.448101:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78a50. 00000400:00000010:3.0:1788971797.448108:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081f68. 00000100:00000200:3.0:1788971797.448114:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063936, portal 10 00000400:00000010:3.0:1788971797.448121:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971797.448123:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179063936, offset 0 00000400:00000200:3.0:1788971797.448132:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971797.448140:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.448148:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971797.448167:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971797.448171:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971797.448175:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971797.448177:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971797.448181:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [285:285:256:4294967295] 192.168.203.124@tcp LPNI seq info [285:285:8:4294967295] 00000400:00000200:3.0:1788971797.448185:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.448195:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971797.448201:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971797.448206:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:3.0:1788971797.448212:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971797.448219:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971797.448223:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971797.448236:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788971797.448241:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a420. 00000100:00000010:3.0:1788971797.448256:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4859a4e0. 02000000:00000010:3.0:1788971797.448463:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016400. 00000400:00000200:3.0:1788971797.448468:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788971797.448470:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78528. 00000400:00000010:3.0:1788971797.448474:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480812a8. 00000100:00000200:3.0:1788971797.448476:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179063808, portal 25 00000400:00000010:3.0:1788971797.448480:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480817f8. 00000100:00000200:3.0:1788971797.448482:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179063808, offset 0 00000400:00000200:3.0:1788971797.448486:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788971797.448490:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.448494:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788971797.448496:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971797.448498:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788971797.448500:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971797.448502:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788971797.448505:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [286:286:255:4294967295] 192.168.203.124@tcp LPNI seq info [286:286:7:4294967295] 00000400:00000200:3.0:1788971797.448508:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.448519:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788971797.448524:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788971797.448527:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d100. 00000800:00000200:3.0:1788971797.448531:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788971797.448535:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788971797.448537:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788971797.448541:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971797.448587:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971797.448590:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:3.0:1788971797.448660:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971797.448666:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971797.448672:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480816e8 00000400:00000010:3.0:1788971797.448675:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480816e8. 00000100:00000200:3.0:1788971797.448683:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6680 x1875872179063936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971797.448762:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971797.448765:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d100. 00000400:00000200:3.0:1788971797.448768:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971797.448771:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971797.448774:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480817f8 00000400:00000010:3.0:1788971797.448776:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480817f8. 00000100:00000200:3.0:1788971797.448780:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4380 x1875872179063808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971813 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971797.456218:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.456228:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971797.456235:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.456241:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.456246:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.456249:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08c00 00000400:00000200:3.0:1788971797.456255:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.456261:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xfa5 [1] + 224 00000400:00000200:3.0:1788971797.456288:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971797.456292:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971797.456336:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4380 x1875872179063808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971813 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971797.456444:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78528. 00000400:00000200:2.0:1788971797.456450:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480812a8 00000400:00000010:2.0:1788971797.456453:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480812a8. 00000100:00000200:2.0:1788971797.456460:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4380 x1875872179063808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971813 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971797.456470:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4380 x1875872179063808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971813 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971797.456496:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016400. 02000000:00000010:2.0:1788971797.456502:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1da00. 00000100:00000010:2.0:1788971797.456506:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4380. 00000400:00000200:3.0:1788971797.457666:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.457672:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971797.457679:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.457682:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.457686:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.457689:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08c80 00000400:00000200:3.0:1788971797.457694:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971797.457700:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xf95 [1] + 224 00000400:00000200:3.0:1788971797.457712:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971797.457716:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971797.457726:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6680 x1875872179063936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971813 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971797.457955:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78a50. 00000400:00000200:2.0:1788971797.457961:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081f68 00000400:00000010:2.0:1788971797.457963:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081f68. 00000100:00000200:2.0:1788971797.457981:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6680 x1875872179063936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971813 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971797.457989:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6680 x1875872179063936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971813 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971797.458013:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017400. 02000000:00000010:2.0:1788971797.458018:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d300. 00000100:00000010:2.0:1788971797.458022:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6680. 00000100:00000010:2.0:1788971798.151419:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a420. 00000100:00000010:3.0:1788971798.151563:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4859a4e0. 00000100:00000010:3.0:1788971802.567811:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788971802.567833:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d100. 02000000:00000010:2.0:1788971802.567926:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aaa00. 00000400:00000200:2.0:1788971802.567947:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971802.567952:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788971802.567960:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788971802.567967:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064064, portal 25 00000400:00000010:2.0:1788971802.567973:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0088. 00000100:00000200:2.0:1788971802.567977:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179064064, offset 0 00000400:00000200:2.0:1788971802.567985:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971802.567994:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.568002:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971802.568022:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971802.568026:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971802.568029:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971802.568032:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971802.568036:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [287:287:256:4294967295] 192.168.203.124@tcp LPNI seq info [287:287:8:4294967295] 00000400:00000200:2.0:1788971802.568040:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.568278:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971802.568286:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971802.568292:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ec00. 00000800:00000200:2.0:1788971802.568329:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971802.568339:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971802.568343:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ec00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971802.568374:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971802.568379:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7a20. 00000100:00000010:2.0:1788971802.568699:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f74e0. 00000100:00000010:3.0:1788971802.569095:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7800. 02000000:00000010:3.0:1788971802.569114:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 02000000:00000010:2.0:1788971802.570342:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000. 00000400:00000200:2.0:1788971802.570354:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971802.570358:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971802.570364:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0000. 00000100:00000200:2.0:1788971802.570370:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064192, portal 10 00000400:00000010:2.0:1788971802.570376:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0bb0. 00000100:00000200:2.0:1788971802.570379:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179064192, offset 0 00000400:00000200:2.0:1788971802.570447:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971802.570456:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.570465:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971802.570470:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971802.570474:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971802.570477:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971802.570481:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971802.570484:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [288:288:255:4294967295] 192.168.203.124@tcp LPNI seq info [288:288:7:4294967295] 00000400:00000200:2.0:1788971802.570488:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.570497:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971802.570502:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971802.570508:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e100. 00000800:00000200:2.0:1788971802.570515:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971802.570728:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971802.570734:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971802.570743:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971802.570807:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971802.570812:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ec00. 00000400:00000200:2.0:1788971802.570817:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971802.570822:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971802.570827:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0088 00000400:00000010:2.0:1788971802.570842:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0088. 00000100:00000200:2.0:1788971802.570852:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179064064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971802.570928:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971802.570930:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e100. 00000400:00000200:2.0:1788971802.571564:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971802.571571:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971802.571575:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0bb0 00000400:00000010:2.0:1788971802.571577:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0bb0. 00000100:00000200:2.0:1788971802.571583:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7800 x1875872179064192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971818 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971802.576378:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971802.576387:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788971802.576395:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971802.576400:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971802.576405:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971802.576409:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08d80 00000400:00000200:3.0:1788971802.576414:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788971802.576420:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xfc5 [1] + 224 00000400:00000200:3.0:1788971802.576446:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971802.576451:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788971802.576464:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7800 x1875872179064192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971818 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971802.576769:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788971802.576773:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0000 00000400:00000010:2.0:1788971802.576775:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0000. 00000100:00000200:2.0:1788971802.576780:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7800 x1875872179064192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971818 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971802.576786:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7800 x1875872179064192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971818 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971802.576812:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000. 02000000:00000010:2.0:1788971802.576816:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:2.0:1788971802.576819:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7800. 00000400:00000200:2.0:1788971802.578842:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.578851:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971802.578859:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.578864:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.578868:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.578872:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08d00 00000400:00000200:2.0:1788971802.578877:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971802.578883:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xfb5 [1] + 224 00000400:00000200:2.0:1788971802.578897:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971802.578922:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971802.578935:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179064064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971818 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971802.579335:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:3.0:1788971802.579341:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660 00000400:00000010:3.0:1788971802.579344:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660. 00000100:00000200:3.0:1788971802.579351:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179064064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971818 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971802.579372:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179064064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971818 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971802.580041:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aaa00. 02000000:00000010:3.0:1788971802.580050:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d100. 00000100:00000010:3.0:1788971802.580054:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000100:00000010:3.0:1788971803.272314:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f74e0. 00000100:00000010:2.0:1788971803.272366:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7a20. 00000100:00000010:2.0:1788971807.687487:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788971807.687507:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e100. 00000100:00000010:2.0:1788971807.687781:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411c00. 02000000:00000010:2.0:1788971807.687864:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ec00. 02000000:00000010:2.0:1788971807.688085:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000. 00000400:00000200:2.0:1788971807.688095:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971807.688100:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788971807.688106:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0000. 00000100:00000200:2.0:1788971807.688114:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064448, portal 10 00000400:00000010:2.0:1788971807.688121:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0bb0. 00000100:00000200:2.0:1788971807.688124:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179064448, offset 0 00000400:00000200:2.0:1788971807.688132:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971807.688140:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.688147:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971807.688152:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971807.688156:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971807.688159:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971807.688161:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971807.688164:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [289:289:256:4294967295] 192.168.203.124@tcp LPNI seq info [289:289:8:4294967295] 00000400:00000200:2.0:1788971807.688169:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.688178:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971807.688185:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971807.688189:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e500. 00000800:00000200:2.0:1788971807.688195:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971807.688203:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971807.688208:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971807.688235:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971807.688240:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7a20. 00000100:00000010:2.0:1788971807.688252:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7f60. 02000000:00000010:2.0:1788971807.688276:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab800. 00000400:00000200:2.0:1788971807.688280:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971807.688283:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b37e8. 00000400:00000010:2.0:1788971807.688285:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0088. 00000100:00000200:2.0:1788971807.688288:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064320, portal 25 00000400:00000010:2.0:1788971807.688291:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0908. 00000100:00000200:2.0:1788971807.688293:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179064320, offset 0 00000400:00000200:2.0:1788971807.688298:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971807.688332:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.688336:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971807.688338:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971807.688340:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971807.688342:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971807.688344:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971807.688347:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [290:290:255:4294967295] 192.168.203.124@tcp LPNI seq info [290:290:7:4294967295] 00000400:00000200:2.0:1788971807.688350:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.688356:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971807.688360:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971807.688364:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e200. 00000800:00000200:2.0:1788971807.688368:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971807.688372:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971807.688374:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971807.688378:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788971807.688482:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971807.688487:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e500. 00000400:00000200:3.0:1788971807.688493:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971807.688498:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971807.688504:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0bb0 00000400:00000010:3.0:1788971807.688506:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0bb0. 00000100:00000200:3.0:1788971807.688515:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411c00 x1875872179064448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971823 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971807.688873:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971807.688877:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e200. 00000400:00000200:3.0:1788971807.688880:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971807.688883:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971807.688886:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0908 00000400:00000010:3.0:1788971807.688888:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0908. 00000100:00000200:3.0:1788971807.688892:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43412d80 x1875872179064320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971823 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971807.692774:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.692784:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971807.692805:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.692810:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.692815:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.692832:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08e80 00000400:00000200:2.0:1788971807.692838:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.692844:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0xfd5 [1] + 224 00000400:00000200:2.0:1788971807.692863:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971807.692867:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971807.692882:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411c00 x1875872179064448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971823 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971807.693385:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:3.0:1788971807.693391:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0000 00000400:00000010:3.0:1788971807.693394:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0000. 00000100:00000200:3.0:1788971807.693401:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411c00 x1875872179064448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971823 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971807.693411:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411c00 x1875872179064448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971823 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971807.693455:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000. 02000000:00000010:3.0:1788971807.693758:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ec00. 00000100:00000010:3.0:1788971807.693767:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411c00. 00000400:00000200:2.0:1788971807.693894:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.693909:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971807.693915:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.693919:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.693923:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.693926:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08e00 00000400:00000200:2.0:1788971807.693931:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971807.693935:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xfe5 [1] + 224 00000400:00000200:2.0:1788971807.693947:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971807.693950:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971807.693958:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43412d80 x1875872179064320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971823 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788971807.694698:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b37e8. 00000400:00000200:3.0:1788971807.694705:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0088 00000400:00000010:3.0:1788971807.694707:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0088. 00000100:00000200:3.0:1788971807.694715:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43412d80 x1875872179064320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971823 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971807.694724:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43412d80 x1875872179064320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971823 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971807.694740:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab800. 02000000:00000010:3.0:1788971807.694743:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e100. 00000100:00000010:3.0:1788971807.694746:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412d80. 00000100:00000010:2.0:1788971808.391455:0:65400:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7a20. 00000100:00000010:3.0:1788971808.393036:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7f60. 00000100:00000010:3.0:1788971812.807504:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6a00. 02000000:00000010:3.0:1788971812.807534:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d100. 02000000:00000010:2.0:1788971812.807609:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab200. 00000400:00000200:2.0:1788971812.807619:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971812.807623:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3e70. 00000400:00000010:2.0:1788971812.807631:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0088. 00000100:00000200:2.0:1788971812.807664:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064576, portal 25 00000400:00000010:2.0:1788971812.807670:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0000. 00000100:00000200:2.0:1788971812.807673:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179064576, offset 0 00000400:00000200:2.0:1788971812.807682:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971812.807698:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.807705:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971812.807721:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971812.807726:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971812.807741:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971812.807744:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971812.807748:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [291:291:256:4294967295] 192.168.203.124@tcp LPNI seq info [291:291:8:4294967295] 00000400:00000200:2.0:1788971812.807752:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.807762:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971812.807768:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971812.807773:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ef00. 00000800:00000200:2.0:1788971812.807779:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971812.807786:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971812.807790:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ef00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971812.807804:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788971812.807808:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7a20. 00000100:00000010:2.0:1788971812.807819:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f7360. 00000100:00000010:3.0:1788971812.807849:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788971812.807855:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d500. 02000000:00000010:2.0:1788971812.807889:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa000. 00000400:00000200:2.0:1788971812.807894:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788971812.807896:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3268. 00000400:00000010:2.0:1788971812.807899:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0908. 00000100:00000200:2.0:1788971812.807902:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179064704, portal 10 00000400:00000010:2.0:1788971812.807907:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0bb0. 00000100:00000200:2.0:1788971812.807909:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179064704, offset 0 00000400:00000200:2.0:1788971812.807913:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788971812.807916:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.807919:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788971812.807922:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971812.807924:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788971812.807926:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971812.807928:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788971812.807931:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [292:292:255:4294967295] 192.168.203.124@tcp LPNI seq info [292:292:7:4294967295] 00000400:00000200:2.0:1788971812.807947:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.807958:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788971812.807963:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788971812.807967:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e300. 00000800:00000200:2.0:1788971812.807970:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788971812.807975:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788971812.807977:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971812.807984:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788971812.808026:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971812.808030:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ef00. 00000400:00000200:2.0:1788971812.808035:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971812.808040:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971812.808044:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0000 00000400:00000010:2.0:1788971812.808046:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0000. 00000100:00000200:2.0:1788971812.808055:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6a00 x1875872179064576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788971812.808175:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971812.808178:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e300. 00000400:00000200:2.0:1788971812.808195:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971812.808198:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788971812.808210:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0bb0 00000400:00000010:2.0:1788971812.808212:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0bb0. 00000100:00000200:2.0:1788971812.808217:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179064704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971828 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971812.820676:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820685:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971812.820696:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820701:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820706:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820710:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e08f00 00000400:00000200:2.0:1788971812.820716:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820722:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0xff5 [1] + 224 00000400:00000200:2.0:1788971812.820748:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971812.820753:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788971812.820766:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6a00 x1875872179064576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971828 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971812.820805:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820809:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788971812.820814:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820816:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820819:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820822:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e08f80 00000400:00000200:2.0:1788971812.820826:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788971812.820832:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1005 [1] + 224 00000400:00000200:2.0:1788971812.820845:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1788971812.820849:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3e70. 00000400:00000200:2.0:1788971812.820849:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788971812.820857:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0088 00000400:00000010:3.0:1788971812.820860:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0088. 00000100:00000200:3.0:1788971812.820868:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6a00 x1875872179064576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971828 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971812.820870:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179064704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971828 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971812.820893:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6a00 x1875872179064576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971828 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971812.820919:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab200. 02000000:00000010:3.0:1788971812.820924:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d100. 00000100:00000010:3.0:1788971812.820928:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6a00. 00000400:00000010:3.0:1788971812.820944:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3268. 00000400:00000200:3.0:1788971812.820947:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0908 00000400:00000010:3.0:1788971812.820949:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0908. 00000100:00000200:3.0:1788971812.820953:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179064704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971828 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788971812.820961:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179064704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971828 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788971812.820977:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa000. 02000000:00000010:3.0:1788971812.820980:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d500. 00000100:00000010:3.0:1788971812.820982:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000100:00000010:2.0:1788971813.511562:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7a20. 00000100:00000010:3.0:1788971813.513005:0:73073:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f7360. 00000001:02000400:1.0:1788971814.480964:0:109560:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_290: @@@@@@ FAIL: Failed to unload modules before test execution Debug log: 3096 lines, 3096 kept, 0 dropped, 0 bad.