00000100:00000010:0.0F:1788971978.002203:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b100. 02000000:00000010:0.0:1788971978.002232:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be200. 00000100:00000010:0.0:1788971978.002261:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971978.002266:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09bef00. 00000100:00000010:0.0:1788971978.002271:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788971978.002275:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d09be000. 02000000:00000010:0.0:1788971978.002321:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003800. 00000400:00000200:0.0:1788971978.002331:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971978.002335:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788971978.002341:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788971978.002349:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576064, portal 25 00000400:00000010:0.0:1788971978.002354:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8aa0. 00000100:00000200:0.0:1788971978.002356:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189576064, offset 0 00000400:00000200:0.0:1788971978.002364:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971978.002373:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971978.002381:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971978.002386:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971978.002390:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971978.002394:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971978.002396:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971978.002400:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [492:492:256:4294967295] 192.168.203.150@tcp LPNI seq info [492:492:8:4294967295] 00000400:00000200:0.0:1788971978.002404:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971978.002413:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971978.002419:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971978.002425:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09be700. 00000800:00000200:0.0:1788971978.002433:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971978.002441:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971978.002446:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09be700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971978.002460:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788971978.002465:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5420. 00000100:00000010:0.0:1788971978.002475:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d20. 02000000:00000010:0.0:1788971978.002487:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003e00. 00000400:00000200:0.0:1788971978.002498:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971978.002501:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788971978.002505:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788971978.002509:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576320, portal 10 00000400:00000010:0.0:1788971978.002514:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8330. 00000100:00000200:0.0:1788971978.002516:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189576320, offset 0 00000400:00000200:0.0:1788971978.002522:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971978.002526:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971978.002530:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971978.002533:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971978.002536:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971978.002538:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971978.002540:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971978.002542:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [493:493:255:4294967295] 192.168.203.150@tcp LPNI seq info [493:493:7:4294967295] 00000400:00000200:0.0:1788971978.002546:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971978.002552:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971978.002556:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971978.002560:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d09bed00. 00000800:00000200:0.0:1788971978.002564:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971978.002568:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971978.002571:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d09bed00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971978.002576:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788971978.002594:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003200. 00000400:00000200:0.0:1788971978.002598:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971978.002600:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788971978.002605:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8f68. 00000100:00000200:0.0:1788971978.002608:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576192, portal 10 00000400:00000010:0.0:1788971978.002612:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788971978.002615:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189576192, offset 0 00000400:00000200:0.0:1788971978.002620:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971978.002624:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971978.002636:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971978.002639:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971978.002642:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971978.002644:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971978.002647:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971978.002650:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [494:494:254:4294967295] 192.168.203.150@tcp LPNI seq info [494:494:6:4294967295] 00000400:00000200:0.0:1788971978.002654:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971978.002661:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971978.002665:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971978.002671:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffe00. 00000800:00000200:0.0:1788971978.002675:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971978.002679:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971978.002681:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffe00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971978.002686:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0F:1788971978.005114:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971978.005131:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09be700. 00000400:00000200:3.0:1788971978.005139:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971978.005147:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971978.005154:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8aa0 00000400:00000010:3.0:1788971978.005157:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8aa0. 00000100:00000200:3.0:1788971978.005170:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b100 x1875872189576064/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971978.005199:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971978.005202:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d09bed00. 00000400:00000200:3.0:1788971978.005205:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971978.005209:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971978.005212:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8330 00000400:00000010:3.0:1788971978.005214:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8330. 00000100:00000200:3.0:1788971978.005220:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189576320/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971978.005286:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971978.005290:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffe00. 00000400:00000200:3.0:1788971978.005292:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971978.005296:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971978.005299:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550 00000400:00000010:3.0:1788971978.005301:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550. 00000100:00000200:3.0:1788971978.005306:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88700 x1875872189576192/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0F:1788971978.017799:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018512:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971978.018524:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018530:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018535:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018539:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f480 00000400:00000200:2.0:1788971978.018544:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018550:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x185d [1] + 224 00000400:00000200:2.0:1788971978.018563:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971978.018568:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971978.018582:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189576320/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971978.018621:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018625:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971978.018629:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018631:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018633:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018636:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f400 00000400:00000200:2.0:1788971978.018639:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018643:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1865 [1] + 224 00000400:00000200:2.0:1788971978.018648:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971978.018651:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971978.018656:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88700 x1875872189576192/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971978.018669:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018672:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971978.018675:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018678:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018680:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018682:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780f380 00000400:00000200:2.0:1788971978.018686:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971978.018696:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1855 [1] + 224 00000400:00000200:2.0:1788971978.018707:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971978.018710:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971978.018718:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b100 x1875872189576064/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971978.018808:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788971978.018905:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8f68 00000400:00000010:0.0:1788971978.018909:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8f68. 00000100:00000200:0.0:1788971978.018916:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88700 x1875872189576192/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971978.018925:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88700 x1875872189576192/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971978.019026:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788971978.019030:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788971978.019032:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788971978.019037:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b100 x1875872189576064/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971978.019045:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b100 x1875872189576064/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971978.019056:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788971978.019066:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:0.0:1788971978.019067:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788971978.019072:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189576320/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971978.019080:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189576320/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971994 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971978.019098:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003200. 02000000:00000010:0.0:1788971978.019105:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09bef00. 00000100:00000010:0.0:1788971978.019109:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971978.019112:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003800. 02000000:00000010:0.0:1788971978.019115:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be200. 00000100:00000010:0.0:1788971978.019117:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b100. 02000000:00000010:0.0:1788971978.019120:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003e00. 02000000:00000010:0.0:1788971978.019121:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d09be000. 00000100:00000010:0.0:1788971978.019123:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 00000100:00000010:2.0:1788971979.029474:0:109717:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5420. 00000100:00000010:3.0:1788971979.029671:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d20. 00000001:02000400:3.0:1788971981.825188:0:114469:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 303: Check peer NI health after link down ========================================================== 12:39:40 (1788971980) 00000020:00000010:3.0:1788971982.406489:0:114540:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971982.406596:0:114540:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:1.0F:1788971982.473929:0:114545:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788971982.474021:0:114545:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000100:00000010:0.0:1788971983.122010:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788971983.122036:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788971983.122081:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b100. 02000000:00000010:1.0:1788971983.122383:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788971983.122396:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971983.122400:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000010:1.0:1788971983.122407:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788971983.122414:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576448, portal 25 00000400:00000010:1.0:1788971983.122419:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788971983.122422:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189576448, offset 0 00000400:00000200:1.0:1788971983.122431:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971983.122439:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971983.122446:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971983.122451:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971983.122455:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971983.122459:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971983.122462:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971983.122466:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [495:495:256:4294967295] 192.168.203.150@tcp LPNI seq info [495:495:8:4294967295] 00000400:00000200:1.0:1788971983.122470:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971983.122480:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971983.122486:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971983.122491:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b100. 00000800:00000200:1.0:1788971983.122498:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971983.122505:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971983.122509:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971983.122525:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971983.122530:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467240. 00000100:00000010:1.0:1788971983.122541:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74679c0. 02000000:00000010:0.0:1788971983.122762:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffb00. 00000100:00000010:0.0:1788971983.122791:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788971983.122797:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff100. 02000000:00000010:1.0:1788971983.123045:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:1.0:1788971983.123051:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971983.123054:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788971983.123058:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8. 00000100:00000200:1.0:1788971983.123062:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576576, portal 10 00000400:00000010:1.0:1788971983.123080:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525330. 00000100:00000200:1.0:1788971983.123096:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189576576, offset 0 00000400:00000200:1.0:1788971983.123102:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971983.123107:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971983.123111:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971983.123114:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971983.123236:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971983.123239:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971983.123241:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971983.123244:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [496:496:255:4294967295] 192.168.203.150@tcp LPNI seq info [496:496:7:4294967295] 00000400:00000200:1.0:1788971983.123248:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971983.123255:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971983.123259:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971983.123263:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:1.0:1788971983.123267:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971983.123271:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971983.123274:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971983.123280:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788971983.123296:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000400:00000200:1.0:1788971983.123299:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971983.123301:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000010:1.0:1788971983.123303:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000800:00000010:3.0:1788971983.123305:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b100. 00000400:00000010:1.0:1788971983.123306:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971983.123309:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576704, portal 10 00000400:00000200:3.0:1788971983.123310:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1788971983.123313:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525cc0. 00000400:00000200:3.0:1788971983.123315:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:1.0:1788971983.123315:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189576704, offset 0 00000400:00000200:1.0:1788971983.123319:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971983.123323:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971983.123325:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220 00000400:00000200:1.0:1788971983.123326:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000010:3.0:1788971983.123327:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220. 00000400:00000200:1.0:1788971983.123329:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971983.123331:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971983.123333:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000100:00000200:3.0:1788971983.123335:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189576448/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788971983.123339:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971983.123342:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [497:497:254:4294967295] 192.168.203.150@tcp LPNI seq info [497:497:6:4294967295] 00000400:00000200:1.0:1788971983.123345:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971983.123359:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971983.123361:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:3.0:1788971983.123364:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971983.123367:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971983.131567:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.131577:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:1.0:1788971983.133378:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971983.133390:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971983.133399:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:1.0:1788971983.133406:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971983.133413:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971983.133418:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971983.133430:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971983.135960:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525330 00000400:00000010:3.0:1788971983.135971:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525330. 00000100:00000200:3.0:1788971983.135986:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b100 x1875872189576576/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971983.136102:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971983.136106:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:3.0:1788971983.136111:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971983.136125:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971983.136129:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525cc0 00000400:00000010:3.0:1788971983.136131:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525cc0. 00000100:00000200:3.0:1788971983.136137:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88700 x1875872189576704/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971983.142745:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142755:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142770:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f580 00000400:00000200:2.0:1788971983.142775:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142790:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1875 [1] + 224 00000400:00000200:2.0:1788971983.142830:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971983.142836:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971983.142852:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b100 x1875872189576576/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971983.142884:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142888:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971983.142897:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142900:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142903:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142905:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780f500 00000400:00000200:2.0:1788971983.142909:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.142913:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x186d [1] + 224 00000400:00000200:2.0:1788971983.142919:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971983.142921:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971983.142928:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189576448/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971983.147466:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788971983.147479:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8 00000400:00000010:1.0:1788971983.147482:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8. 00000400:00000200:2.0:1788971983.150745:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.150753:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971983.150763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.150767:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.150772:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.150787:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f600 00000400:00000200:2.0:1788971983.150793:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971983.150800:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x187d [1] + 224 00000400:00000200:2.0:1788971983.150829:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971983.150833:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971983.150845:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88700 x1875872189576704/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971983.151182:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b100 x1875872189576576/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971983.151195:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b100 x1875872189576576/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971983.151230:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:1.0:1788971983.151238:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffb00. 00000100:00000010:1.0:1788971983.151242:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b100. 00000400:00000010:1.0:1788971983.151253:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971983.151256:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:1.0:1788971983.151258:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971983.151263:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88700 x1875872189576704/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971983.151271:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88700 x1875872189576704/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971983.151283:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:1.0:1788971983.151286:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff100. 00000100:00000010:1.0:1788971983.151304:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700. 00000400:00000010:0.0:1788971983.153634:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8. 00000400:00000200:0.0:1788971983.153650:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:0.0:1788971983.153652:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:0.0:1788971983.153667:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189576448/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971983.153679:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189576448/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971999 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971983.153704:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:0.0:1788971983.153710:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788971983.153729:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 00000020:00000010:3.0:1788971983.506753:0:114549:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971983.507179:0:114549:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:2.0:1788971984.145957:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467240. 00000100:00000010:3.0:1788971984.146453:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74679c0. 00000020:00000010:2.0:1788971984.565209:0:114553:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788971984.565303:0:114553:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:3.0:1788971985.603201:0:114558:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971985.603461:0:114558:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:2.0:1788971986.641874:0:114562:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788971986.641986:0:114562:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:1.0:1788971987.674141:0:114566:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788971987.674260:0:114566:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000100:00000010:3.0:1788971988.242391:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788971988.242414:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788971988.242448:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100. 02000000:00000010:3.0:1788971988.242453:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788971988.242459:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788971988.242462:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 02000000:00000010:3.0:1788971988.242510:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788971988.242519:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788971988.242523:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788971988.242530:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788971988.242537:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576960, portal 10 00000400:00000010:3.0:1788971988.242547:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788971988.242552:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189576960, offset 0 02000000:00000010:2.0:1788971988.242552:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b800. 00000400:00000200:3.0:1788971988.242566:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971988.242572:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971988.242579:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126840. 00000400:00000200:3.0:1788971988.242581:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788971988.242591:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000400:00000200:3.0:1788971988.242593:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000100:00000200:2.0:1788971988.242601:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189576832, portal 25 00000400:00000200:3.0:1788971988.242602:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788971988.242613:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000010:2.0:1788971988.242613:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550. 00000100:00000200:2.0:1788971988.242622:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189576832, offset 0 00000400:00000200:3.0:1788971988.242624:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971988.242632:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971988.242639:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788971988.242641:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [498:498:256:4294967295] 192.168.203.150@tcp LPNI seq info [498:498:8:4294967295] 00000400:00000200:3.0:1788971988.242649:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.242658:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788971988.242664:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971988.242668:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:3.0:1788971988.242671:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00. 00000400:00000200:2.0:1788971988.242682:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000800:00000200:3.0:1788971988.242683:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:2.0:1788971988.242690:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:3.0:1788971988.242695:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:2.0:1788971988.242697:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000800:00000200:3.0:1788971988.242706:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971988.242707:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788971988.242725:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788971988.242731:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09480. 00000400:00000200:2.0:1788971988.242738:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971988.242753:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [499:499:255:4294967295] 192.168.203.150@tcp LPNI seq info [499:499:7:4294967295] 00000100:00000010:3.0:1788971988.242755:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09b40. 00000400:00000200:2.0:1788971988.242762:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.242777:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971988.242784:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971988.242792:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d100. 00000800:00000200:2.0:1788971988.242799:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971988.242806:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971988.242811:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971988.242881:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788971988.242904:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094aa00. 00000800:00000200:3.0:1788971988.242907:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:2.0:1788971988.242910:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:3.0:1788971988.242914:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00. 00000400:00000010:2.0:1788971988.242916:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126cb8. 00000400:00000200:3.0:1788971988.242923:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1788971988.242923:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971988.242931:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577088, portal 10 00000400:00000200:3.0:1788971988.242939:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000010:2.0:1788971988.242942:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000400:00000200:3.0:1788971988.242947:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908 00000400:00000010:3.0:1788971988.242952:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788971988.242973:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793100 x1875872189576960/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971988.243004:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971988.243007:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d100. 00000400:00000200:3.0:1788971988.243011:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971988.243014:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971988.247667:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189577088, offset 0 00000400:00000200:2.0:1788971988.247700:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788971988.247716:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247726:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788971988.247732:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788971988.247737:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788971988.247750:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788971988.247754:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788971988.247758:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [500:500:255:4294967295] 192.168.203.150@tcp LPNI seq info [500:500:7:4294967295] 00000400:00000200:2.0:1788971988.247763:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247774:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788971988.247781:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788971988.247789:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d500. 00000800:00000200:2.0:1788971988.247797:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788971988.247840:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788971988.247846:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788971988.247858:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247927:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247933:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971988.247939:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247943:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247947:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247951:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f700 00000400:00000200:2.0:1788971988.247956:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971988.247963:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1885 [1] + 224 00000400:00000200:2.0:1788971988.247980:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971988.247985:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971988.247999:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793100 x1875872189576960/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971988.248052:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:2.0:1788971988.248058:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:2.0:1788971988.248061:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:2.0:1788971988.248066:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793100 x1875872189576960/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971988.248075:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793100 x1875872189576960/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971988.248116:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:2.0:1788971988.248124:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788971988.248129:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100. 00000400:00000200:3.0:1788971988.254447:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550 00000400:00000010:3.0:1788971988.254454:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550. 00000100:00000200:3.0:1788971988.254469:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792300 x1875872189576832/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971988.254505:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.254509:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971988.254515:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.254519:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.254523:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.254527:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780f680 00000400:00000200:3.0:1788971988.254531:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.254537:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1895 [1] + 224 00000400:00000200:3.0:1788971988.254548:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971988.254552:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971988.254559:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792300 x1875872189576832/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788971988.254637:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971988.254641:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d500. 00000400:00000200:3.0:1788971988.254645:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971988.254649:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971988.254652:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:3.0:1788971988.254654:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:3.0:1788971988.254659:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189577088/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971988.258636:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126840. 00000400:00000200:2.0:1788971988.258652:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:2.0:1788971988.258656:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788971988.258674:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792300 x1875872189576832/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971988.258717:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792300 x1875872189576832/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971988.258764:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b800. 02000000:00000010:2.0:1788971988.258774:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:2.0:1788971988.258780:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300. 00000400:00000200:3.0:1788971988.259844:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.259854:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971988.259863:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.259868:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.259874:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.259878:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f780 00000400:00000200:3.0:1788971988.259883:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971988.259902:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18a5 [1] + 224 00000400:00000200:3.0:1788971988.259916:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971988.259921:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971988.259933:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189577088/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788971988.260329:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126cb8. 00000400:00000200:2.0:1788971988.260335:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:2.0:1788971988.260338:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788971988.260345:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189577088/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788971988.260355:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189577088/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972004 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788971988.260383:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094aa00. 02000000:00000010:2.0:1788971988.260390:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788971988.260394:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 00000020:00000010:3.0:1788971988.723304:0:114570:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971988.725202:0:114570:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:3.0:1788971989.265938:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09b40. 00000100:00000010:2.0:1788971989.265938:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09480. 00000020:00000010:1.0:1788971989.763847:0:114574:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788971989.763985:0:114574:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788971990.804240:0:114578:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788971990.804832:0:114578:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:2.0:1788971991.832329:0:114582:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788971991.832442:0:114582:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:0.0:1788971992.878313:0:114587:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788971992.878423:0:114587:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000100:00000010:0.0:1788971993.362902:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788971993.362939:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 02000000:00000010:1.0:1788971993.363131:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788971993.363145:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971993.363150:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788971993.363160:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971993.363171:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577216, portal 25 00000400:00000010:1.0:1788971993.363177:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8. 00000100:00000200:1.0:1788971993.363181:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189577216, offset 0 00000400:00000200:1.0:1788971993.363193:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971993.363205:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971993.363214:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971993.363219:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971993.363224:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971993.363229:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971993.363232:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971993.363236:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [501:501:256:4294967295] 192.168.203.150@tcp LPNI seq info [501:501:8:4294967295] 00000400:00000200:1.0:1788971993.363241:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971993.363584:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971993.363593:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971993.363601:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:1.0:1788971993.363610:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971993.363620:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971993.363626:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971993.363656:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971993.363664:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467a80. 00000100:00000010:1.0:1788971993.363719:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467120. 00000100:00000010:0.0:1788971993.363907:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788971993.363922:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff800. 00000800:00000200:2.0:1788971993.364106:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971993.364112:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:2.0:1788971993.364120:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971993.364126:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971993.364135:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8 00000400:00000010:2.0:1788971993.364138:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8. 00000100:00000200:2.0:1788971993.364148:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189577216/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971993.364308:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0947a00. 00000400:00000200:1.0:1788971993.364316:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971993.364320:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788971993.364326:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788971993.364332:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577344, portal 10 00000400:00000010:1.0:1788971993.364340:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788971993.364343:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189577344, offset 0 00000400:00000200:1.0:1788971993.364349:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971993.364355:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971993.364360:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971993.364365:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971993.364369:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971993.364374:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971993.364377:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971993.364380:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [502:502:256:4294967295] 192.168.203.150@tcp LPNI seq info [502:502:8:4294967295] 00000400:00000200:1.0:1788971993.364543:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971993.364554:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971993.364561:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971993.364566:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788971993.364581:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971993.364587:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971993.364591:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971993.364613:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971993.364922:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971993.364927:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:3.0:1788971993.364932:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971993.364936:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971993.364941:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68 00000400:00000010:3.0:1788971993.364943:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68. 00000100:00000200:3.0:1788971993.365310:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a300 x1875872189577344/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788971993.369466:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88380. 00000400:00000200:2.0:1788971993.369472:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.369485:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971993.369496:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.369522:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.369529:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.369534:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780f800 00000400:00000200:2.0:1788971993.369539:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.369546:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x18b5 [1] + 224 00000400:00000200:2.0:1788971993.369561:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971993.369566:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971993.369754:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189577216/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971993.369892:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cfff00. 00000400:00000200:2.0:1788971993.370321:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.370327:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971993.370333:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.370336:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.370339:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.370342:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f880 00000400:00000200:2.0:1788971993.370347:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971993.370352:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18bd [1] + 224 00000400:00000200:2.0:1788971993.370365:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971993.370368:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971993.370376:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a300 x1875872189577344/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971993.370839:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003200. 00000400:00000200:0.0:1788971993.370861:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788971993.370866:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788971993.370874:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788971993.370884:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577472, portal 10 00000400:00000010:0.0:1788971993.370891:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788971993.370895:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189577472, offset 0 00000400:00000200:0.0:1788971993.370905:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788971993.370912:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971993.370921:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788971993.370927:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788971993.370931:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788971993.370936:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788971993.370939:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788971993.370943:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [503:503:256:4294967295] 192.168.203.150@tcp LPNI seq info [503:503:8:4294967295] 00000400:00000200:0.0:1788971993.370948:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788971993.370971:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788971993.370978:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788971993.370985:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff000. 00000800:00000200:0.0:1788971993.370992:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788971993.371008:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788971993.371014:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788971993.371050:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971993.371180:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971993.371186:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff000. 00000400:00000200:3.0:1788971993.371192:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971993.371197:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971993.371202:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:3.0:1788971993.371204:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:3.0:1788971993.371219:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88380 x1875872189577472/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971993.373850:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971993.373867:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:1.0:1788971993.373869:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971993.373888:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189577216/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971993.374424:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189577216/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971993.374493:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788971993.374501:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:1.0:1788971993.374507:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 00000400:00000010:1.0:1788971993.374518:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788971993.374521:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:1.0:1788971993.374523:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788971993.374529:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a300 x1875872189577344/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971993.374538:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a300 x1875872189577344/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971993.374557:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0947a00. 02000000:00000010:1.0:1788971993.374562:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff800. 00000100:00000010:1.0:1788971993.374564:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a300. 00000400:00000200:3.0:1788971993.383506:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971993.383519:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971993.383531:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971993.383537:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971993.383544:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971993.383548:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780f900 00000400:00000200:3.0:1788971993.383555:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971993.383563:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18c5 [1] + 224 00000400:00000200:3.0:1788971993.383582:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971993.383587:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971993.383607:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88380 x1875872189577472/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788971993.383778:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788971993.383785:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:0.0:1788971993.383789:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788971993.383798:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88380 x1875872189577472/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788971993.383809:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88380 x1875872189577472/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972009 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788971993.384451:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003200. 02000000:00000010:0.0:1788971993.384463:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cfff00. 00000100:00000010:0.0:1788971993.384469:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88380. 00000020:00000010:1.0:1788971993.935261:0:114591:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788971993.935907:0:114591:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000100:00000010:2.0:1788971994.385990:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467a80. 00000100:00000010:3.0:1788971994.386457:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467120. 00000020:00000010:3.0:1788971994.985675:0:114595:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971994.986293:0:114595:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:2.0:1788971996.019042:0:114599:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788971996.019154:0:114599:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:3.0:1788971997.059144:0:114603:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971997.059381:0:114603:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971998.125158:0:114607:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788971998.125261:0:114607:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:0.0:1788971998.482186:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88380. 02000000:00000010:0.0:1788971998.482208:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cfff00. 02000000:00000010:1.0:1788971998.482616:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788971998.482626:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971998.482630:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788971998.482637:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788971998.482644:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577600, portal 25 00000400:00000010:1.0:1788971998.482649:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788971998.482652:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189577600, offset 0 00000400:00000200:1.0:1788971998.482662:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971998.482679:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971998.482687:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971998.482692:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971998.482696:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971998.482700:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971998.482703:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971998.482706:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [504:504:256:4294967295] 192.168.203.150@tcp LPNI seq info [504:504:8:4294967295] 00000400:00000200:1.0:1788971998.482725:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971998.482735:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971998.482741:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971998.482746:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788971998.482752:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971998.482761:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971998.482765:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971998.482788:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788971998.482793:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74672a0. 00000100:00000010:1.0:1788971998.482805:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467600. 00000800:00000200:2.0:1788971998.482970:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971998.482975:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:2.0:1788971998.482980:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971998.482986:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971998.482991:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:2.0:1788971998.482994:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:2.0:1788971998.483010:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88380 x1875872189577600/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788971998.483559:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a680. 02000000:00000010:0.0:1788971998.483572:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff200. 02000000:00000010:1.0:1788971998.486175:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0947a00. 00000400:00000200:1.0:1788971998.486188:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971998.486193:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788971998.486210:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788971998.486877:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577728, portal 10 00000400:00000010:1.0:1788971998.486886:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788971998.486890:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189577728, offset 0 00000400:00000200:1.0:1788971998.486900:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971998.486911:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971998.486919:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971998.486923:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971998.486928:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971998.486932:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971998.486935:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971998.486938:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [505:505:256:4294967295] 192.168.203.150@tcp LPNI seq info [505:505:8:4294967295] 00000400:00000200:1.0:1788971998.486943:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971998.486952:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971998.486958:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971998.486964:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788971998.486971:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971998.486992:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971998.486997:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971998.487022:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788971998.487200:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788971998.487205:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:3.0:1788971998.487220:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971998.487225:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788971998.487230:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:3.0:1788971998.487233:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:3.0:1788971998.487243:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a680 x1875872189577728/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788971998.489776:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8ad80. 02000000:00000010:0.0:1788971998.489798:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 02000000:00000010:1.0:1788971998.489933:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:1.0:1788971998.490068:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788971998.490073:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788971998.490080:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788971998.490087:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577856, portal 10 00000400:00000010:1.0:1788971998.490094:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788971998.490097:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189577856, offset 0 00000400:00000200:1.0:1788971998.490106:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788971998.490115:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971998.490124:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788971998.490129:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788971998.490133:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788971998.490137:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788971998.490140:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788971998.490143:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [506:506:256:4294967295] 192.168.203.150@tcp LPNI seq info [506:506:8:4294967295] 00000400:00000200:1.0:1788971998.490147:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788971998.490156:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788971998.490162:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788971998.490168:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788971998.490174:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788971998.490183:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788971998.490188:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788971998.490494:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788971998.493487:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788971998.493496:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:2.0:1788971998.493503:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971998.493512:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788971998.493520:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:2.0:1788971998.493523:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000100:00000200:2.0:1788971998.493533:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8ad80 x1875872189577856/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788971998.499774:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971998.499798:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788971998.499808:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971998.499842:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971998.499849:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971998.499853:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780f980 00000400:00000200:3.0:1788971998.499860:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788971998.499867:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x18cd [1] + 224 00000400:00000200:3.0:1788971998.499882:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788971998.499887:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788971998.499902:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88380 x1875872189577600/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971998.500221:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788971998.500237:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:1.0:1788971998.500239:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788971998.500246:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88380 x1875872189577600/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971998.500267:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88380 x1875872189577600/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971998.500293:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788971998.500299:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cfff00. 00000100:00000010:1.0:1788971998.500305:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88380. 00000400:00000200:2.0:1788971998.501782:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501791:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971998.501799:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501803:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501808:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501812:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fa00 00000400:00000200:2.0:1788971998.501841:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501847:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18d5 [1] + 224 00000400:00000200:2.0:1788971998.501857:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971998.501862:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971998.501886:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a680 x1875872189577728/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788971998.501917:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501921:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788971998.501926:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.501928:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.502094:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.502100:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fa80 00000400:00000200:2.0:1788971998.502104:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788971998.502108:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18dd [1] + 224 00000400:00000200:2.0:1788971998.502129:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788971998.502132:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788971998.502140:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8ad80 x1875872189577856/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971998.502380:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788971998.502386:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525088 00000400:00000010:1.0:1788971998.502388:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788971998.502395:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8ad80 x1875872189577856/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971998.502404:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8ad80 x1875872189577856/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788971998.502428:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788971998.502430:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:1.0:1788971998.502432:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788971998.502436:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a680 x1875872189577728/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788971998.502444:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a680 x1875872189577728/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972014 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788971998.502459:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:1.0:1788971998.502465:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:1.0:1788971998.502469:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8ad80. 02000000:00000010:1.0:1788971998.502473:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0947a00. 02000000:00000010:1.0:1788971998.502476:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff200. 00000100:00000010:1.0:1788971998.502478:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a680. 00000020:00000010:2.0:1788971999.197276:0:114611:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788971999.197367:0:114611:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000100:00000010:2.0:1788971999.507970:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74672a0. 00000100:00000010:3.0:1788971999.508776:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467600. 00000020:00000010:3.0:1788972000.234741:0:114616:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972000.234934:0:114616:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:1.0:1788972001.269917:0:114620:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972001.269998:0:114620:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:2.0:1788972002.312393:0:114624:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972002.312491:0:114624:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:3.0:1788972003.348045:0:114628:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972003.348144:0:114628:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:0.0:1788972003.602436:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89c00. 02000000:00000010:0.0:1788972003.602457:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff600. 00000100:00000010:0.0:1788972003.602493:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8bb80. 02000000:00000010:0.0:1788972003.602498:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffc00. 00000100:00000010:0.0:1788972003.602503:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8aa00. 02000000:00000010:0.0:1788972003.602506:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff300. 02000000:00000010:1.0:1788972003.602714:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:1.0:1788972003.602738:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972003.602743:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972003.602749:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972003.602756:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189577984, portal 25 00000400:00000010:1.0:1788972003.602761:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788972003.602764:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189577984, offset 0 00000400:00000200:1.0:1788972003.602773:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972003.602780:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972003.602788:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972003.602793:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972003.602797:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972003.602801:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972003.602804:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972003.602808:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [507:507:256:4294967295] 192.168.203.150@tcp LPNI seq info [507:507:8:4294967295] 00000400:00000200:1.0:1788972003.602812:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972003.602850:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972003.602856:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972003.602861:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:1.0:1788972003.603048:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972003.603058:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972003.603062:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972003.603080:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972003.603086:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467c00. 00000100:00000010:1.0:1788972003.603095:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467ba0. 02000000:00000010:1.0:1788972003.603106:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788972003.603109:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972003.603111:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788972003.603115:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788972003.603118:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578240, portal 10 00000400:00000010:1.0:1788972003.603122:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525440. 00000100:00000200:1.0:1788972003.603125:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189578240, offset 0 00000400:00000200:1.0:1788972003.603145:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972003.603149:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972003.603152:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972003.603155:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972003.603157:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972003.603160:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972003.603162:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972003.603166:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [508:508:255:4294967295] 192.168.203.150@tcp LPNI seq info [508:508:7:4294967295] 00000400:00000200:1.0:1788972003.603170:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972003.603177:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972003.603181:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972003.603185:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:1.0:1788972003.603189:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972003.603200:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972003.603203:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972003.603209:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972003.603224:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000010:1.0:1788972003.603227:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000800:00000010:3.0:1788972003.603230:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:1.0:1788972003.603234:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.603238:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1788972003.603238:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972003.603244:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972003.603248:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578112, portal 10 00000400:00000200:3.0:1788972003.603252:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000010:1.0:1788972003.603254:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000. 00000400:00000200:3.0:1788972003.603259:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525088 00000400:00000010:3.0:1788972003.603263:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788972003.603268:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189578112, offset 0 00000100:00000200:3.0:1788972003.603274:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89c00 x1875872189577984/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972003.603276:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972003.603283:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972003.603289:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972003.603294:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972003.603296:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972003.603299:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972003.603302:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972003.603305:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [509:509:255:4294967295] 192.168.203.150@tcp LPNI seq info [509:509:7:4294967295] 00000800:00000200:3.0:1788972003.603310:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972003.603314:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:1.0:1788972003.603318:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.603321:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972003.603330:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:1.0:1788972003.603333:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972003.603339:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972003.603342:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525440 00000800:00000010:1.0:1788972003.603343:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000400:00000010:3.0:1788972003.603347:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525440. 00000800:00000200:1.0:1788972003.603351:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000100:00000200:3.0:1788972003.603357:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8aa00 x1875872189578240/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1788972003.603359:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972003.603361:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972003.603368:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972003.603404:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972003.603406:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:3.0:1788972003.603409:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972003.603413:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972003.603416:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000 00000400:00000010:3.0:1788972003.603418:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000. 00000100:00000200:3.0:1788972003.603423:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8bb80 x1875872189578112/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972003.608451:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608462:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972003.608471:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608476:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608481:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608485:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fc00 00000400:00000200:2.0:1788972003.608490:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608496:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18ed [1] + 224 00000400:00000200:2.0:1788972003.608507:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972003.608511:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972003.608524:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8aa00 x1875872189578240/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972003.608559:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608563:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972003.608567:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608569:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608571:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608574:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fb80 00000400:00000200:2.0:1788972003.608577:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972003.608580:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18f5 [1] + 224 00000400:00000200:2.0:1788972003.608598:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972003.608601:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972003.608607:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8bb80 x1875872189578112/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972003.608701:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788972003.608706:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:1.0:1788972003.608709:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972003.608715:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8bb80 x1875872189578112/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972003.608724:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8bb80 x1875872189578112/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972003.608752:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788972003.608754:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:1.0:1788972003.608756:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788972003.608760:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8aa00 x1875872189578240/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972003.608768:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8aa00 x1875872189578240/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972003.608782:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:1.0:1788972003.608787:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffc00. 00000100:00000010:1.0:1788972003.608792:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8bb80. 02000000:00000010:1.0:1788972003.608795:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788972003.608797:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff300. 00000100:00000010:1.0:1788972003.608799:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8aa00. 00000400:00000200:3.0:1788972003.608854:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.608860:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972003.608866:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.608869:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.608872:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.608875:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780fb00 00000400:00000200:3.0:1788972003.608878:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972003.608883:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x18e5 [1] + 224 00000400:00000200:3.0:1788972003.608890:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972003.608894:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972003.608902:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89c00 x1875872189577984/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972003.608990:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788972003.609000:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:1.0:1788972003.609002:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972003.609007:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89c00 x1875872189577984/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972003.609015:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89c00 x1875872189577984/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972019 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972003.609028:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:1.0:1788972003.609031:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff600. 00000100:00000010:1.0:1788972003.609034:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89c00. 00000020:00000010:2.0:1788972004.393714:0:114632:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972004.394098:0:114632:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000100:00000010:2.0:1788972004.626228:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467c00. 00000100:00000010:3.0:1788972004.626394:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467ba0. 00000020:00000010:1.0:1788972005.448483:0:114636:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972005.448574:0:114636:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:3.0:1788972006.488142:0:114640:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972006.488229:0:114640:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972007.532283:0:114645:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972007.532374:0:114645:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:2.0:1788972008.602904:0:114649:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972008.602994:0:114649:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000100:00000010:1.0:1788972008.722204:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3b80. 02000000:00000010:1.0:1788972008.722227:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 00000100:00000010:1.0:1788972008.722264:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800. 02000000:00000010:1.0:1788972008.722270:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 00000100:00000010:1.0:1788972008.722276:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80. 02000000:00000010:1.0:1788972008.722280:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 02000000:00000010:1.0:1788972008.722317:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788972008.722327:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972008.722331:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972008.722337:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972008.722343:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578496, portal 10 00000400:00000010:1.0:1788972008.722350:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788972008.722353:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189578496, offset 0 00000400:00000200:1.0:1788972008.722362:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972008.722370:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972008.722377:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972008.722382:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972008.722385:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972008.722389:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972008.722392:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972008.722409:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [510:510:256:4294967295] 192.168.203.150@tcp LPNI seq info [510:510:8:4294967295] 00000400:00000200:1.0:1788972008.722413:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972008.722423:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972008.722429:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972008.722434:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:1.0:1788972008.722441:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972008.722447:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972008.722452:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972008.722467:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972008.722472:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74673c0. 00000100:00000010:1.0:1788972008.722482:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467b40. 02000000:00000010:1.0:1788972008.722522:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:1.0:1788972008.722527:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972008.722529:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788972008.722532:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972008.722536:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578368, portal 25 00000400:00000010:1.0:1788972008.722539:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972008.722541:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189578368, offset 0 00000400:00000200:1.0:1788972008.722545:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972008.722548:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972008.722552:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972008.722555:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972008.722557:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972008.722560:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972008.722562:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972008.722564:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [511:511:255:4294967295] 192.168.203.150@tcp LPNI seq info [511:511:7:4294967295] 00000400:00000200:1.0:1788972008.722568:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972008.722573:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972008.722578:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972008.722581:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788972008.722590:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972008.722595:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972008.722612:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972008.722618:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972008.722630:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000400:00000200:1.0:1788972008.722633:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972008.722636:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972008.722639:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972008.722642:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578624, portal 10 00000400:00000010:1.0:1788972008.722647:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525990. 00000100:00000200:1.0:1788972008.722649:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189578624, offset 0 00000400:00000200:1.0:1788972008.722653:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972008.722657:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972008.722660:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972008.722662:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972008.722665:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972008.722666:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972008.722669:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972008.722671:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [512:512:254:4294967295] 192.168.203.150@tcp LPNI seq info [512:512:6:4294967295] 00000400:00000200:1.0:1788972008.722675:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972008.722681:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972008.722685:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972008.722688:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972008.722691:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972008.722694:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972008.722697:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972008.722701:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972008.729746:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972008.729756:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:2.0:1788972008.729763:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972008.729772:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972008.729780:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:2.0:1788972008.729782:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:2.0:1788972008.729794:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3800 x1875872189578496/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972008.729852:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972008.729855:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:2.0:1788972008.729858:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972008.729862:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972008.729864:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:2.0:1788972008.729866:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:2.0:1788972008.729871:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3b80 x1875872189578368/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972008.729938:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972008.729941:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:2.0:1788972008.729945:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972008.729948:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972008.729951:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525990 00000400:00000010:2.0:1788972008.729953:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525990. 00000100:00000200:2.0:1788972008.729972:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1f80 x1875872189578624/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972008.739521:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.739534:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972008.739542:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.739547:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.739566:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.739570:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780fc80 00000400:00000200:3.0:1788972008.739576:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.739582:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1905 [1] + 224 00000400:00000200:3.0:1788972008.739612:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972008.739617:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972008.739631:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3b80 x1875872189578368/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972008.739750:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788972008.739755:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:1.0:1788972008.739759:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972008.739765:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3b80 x1875872189578368/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972008.739782:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3b80 x1875872189578368/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972008.739811:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:1.0:1788972008.739845:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:1.0:1788972008.739849:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3b80. 00000400:00000200:2.0:1788972008.740744:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972008.740754:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972008.740760:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972008.740763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972008.740767:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972008.740771:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fd00 00000400:00000200:2.0:1788972008.740775:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972008.740781:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x18fd [1] + 224 00000400:00000200:2.0:1788972008.740791:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972008.740794:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972008.740804:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3800 x1875872189578496/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972008.743547:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.743558:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972008.743565:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.743570:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.743575:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.743580:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fd80 00000400:00000200:3.0:1788972008.743585:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972008.743592:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x190d [1] + 224 00000400:00000200:3.0:1788972008.743617:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972008.743621:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972008.743635:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1f80 x1875872189578624/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972008.746062:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788972008.746074:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:1.0:1788972008.746076:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972008.746089:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3800 x1875872189578496/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972008.746099:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3800 x1875872189578496/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972008.746150:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788972008.746158:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:1.0:1788972008.746162:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800. 00000400:00000010:1.0:1788972008.746171:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788972008.746174:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:1.0:1788972008.746175:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972008.746180:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1f80 x1875872189578624/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972008.746188:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1f80 x1875872189578624/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972024 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972008.746200:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:1.0:1788972008.746203:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:1.0:1788972008.746206:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80. 00000020:00000010:3.0:1788972009.641721:0:114653:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972009.641809:0:114653:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:3.0:1788972009.745933:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467b40. 00000100:00000010:2.0:1788972009.745933:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74673c0. 00000020:00000010:0.0:1788972010.668742:0:114657:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972010.668850:0:114657:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000020:00000010:1.0:1788972011.714462:0:114661:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972011.714553:0:114661:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972012.759386:0:114666:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972012.759475:0:114666:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:3.0:1788972013.810165:0:114670:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972013.810246:0:114670:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:3.0:1788972013.842154:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800. 02000000:00000010:3.0:1788972013.842179:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00. 02000000:00000010:2.0:1788972013.842271:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788972013.842280:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972013.842284:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126cb8. 00000400:00000010:2.0:1788972013.842290:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788972013.842296:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578752, portal 25 00000400:00000010:2.0:1788972013.842301:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972013.842304:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189578752, offset 0 00000400:00000200:2.0:1788972013.842313:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972013.842322:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.842330:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972013.842334:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972013.842338:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972013.842346:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972013.842349:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972013.842353:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [513:513:256:4294967295] 192.168.203.150@tcp LPNI seq info [513:513:8:4294967295] 00000400:00000200:2.0:1788972013.842373:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.842383:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972013.842390:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972013.842395:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d200. 00000800:00000200:2.0:1788972013.842401:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972013.842408:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972013.842412:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972013.842427:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972013.842432:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e240. 00000100:00000010:2.0:1788972013.842443:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e9c0. 00000100:00000010:3.0:1788972013.842476:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972013.842483:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 02000000:00000010:2.0:1788972013.842655:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a600. 00000400:00000200:2.0:1788972013.842661:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972013.842664:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126840. 00000400:00000010:2.0:1788972013.842668:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972013.842671:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189578880, portal 10 00000400:00000010:2.0:1788972013.842677:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18. 00000100:00000200:2.0:1788972013.842679:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189578880, offset 0 00000400:00000200:2.0:1788972013.842683:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972013.842687:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.842690:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972013.842693:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972013.842695:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972013.842698:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972013.842700:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972013.842703:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [514:514:255:4294967295] 192.168.203.150@tcp LPNI seq info [514:514:7:4294967295] 00000400:00000200:2.0:1788972013.842707:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.842712:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972013.842717:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972013.842721:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d500. 00000800:00000200:2.0:1788972013.842725:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972013.842736:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972013.842739:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972013.842745:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972013.842791:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972013.842795:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d200. 00000400:00000200:2.0:1788972013.842799:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972013.842804:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972013.842808:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:2.0:1788972013.842811:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972013.842863:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793800 x1875872189578752/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972013.842926:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972013.842929:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d500. 00000400:00000200:2.0:1788972013.842932:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972013.842935:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972013.842937:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18 00000400:00000010:2.0:1788972013.842939:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18. 00000100:00000200:2.0:1788972013.842943:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189578880/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788972013.843055:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788972013.843067:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300. 02000000:00000010:2.0:1788972013.843104:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788972013.843108:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972013.843110:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126688. 00000400:00000010:2.0:1788972013.843113:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18. 00000100:00000200:2.0:1788972013.843116:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579008, portal 10 00000400:00000010:2.0:1788972013.843120:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972013.843122:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189579008, offset 0 00000400:00000200:2.0:1788972013.843126:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972013.843130:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.843134:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972013.843137:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972013.843139:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972013.843142:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972013.843149:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972013.843152:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [515:515:256:4294967295] 192.168.203.150@tcp LPNI seq info [515:515:8:4294967295] 00000400:00000200:2.0:1788972013.843156:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.843162:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972013.843166:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972013.843169:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d500. 00000800:00000200:2.0:1788972013.843173:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972013.843176:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972013.843179:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972013.843190:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972013.843256:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972013.843258:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d500. 00000400:00000200:2.0:1788972013.843261:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972013.843264:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972013.843267:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:2.0:1788972013.843268:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972013.843274:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792300 x1875872189579008/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972013.846744:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846754:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972013.846763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846768:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846773:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846777:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780fe80 00000400:00000200:2.0:1788972013.846782:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846788:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1925 [1] + 224 00000400:00000200:2.0:1788972013.846799:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972013.846804:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972013.846865:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189578880/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972013.846917:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846920:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972013.846935:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846938:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846941:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846945:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780ff00 00000400:00000200:2.0:1788972013.846948:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846952:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1935 [1] + 224 00000400:00000200:2.0:1788972013.846959:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972013.846962:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972013.846968:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792300 x1875872189579008/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972013.846981:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846984:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972013.846987:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846990:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846992:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.846995:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780fe00 00000400:00000200:2.0:1788972013.846998:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972013.847002:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1915 [1] + 224 00000400:00000200:2.0:1788972013.847008:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972013.847010:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972013.847015:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793800 x1875872189578752/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972013.847370:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126688. 00000400:00000200:3.0:1788972013.847376:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18 00000400:00000010:3.0:1788972013.847378:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18. 00000100:00000200:3.0:1788972013.847388:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792300 x1875872189579008/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972013.847396:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792300 x1875872189579008/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972013.847422:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126840. 00000400:00000200:3.0:1788972013.847425:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68 00000400:00000010:3.0:1788972013.847427:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68. 00000100:00000200:3.0:1788972013.847430:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189578880/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972013.847437:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189578880/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972013.847448:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126cb8. 00000400:00000200:3.0:1788972013.847451:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:3.0:1788972013.847452:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:3.0:1788972013.847455:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793800 x1875872189578752/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972013.847462:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793800 x1875872189578752/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972029 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972013.847475:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:3.0:1788972013.847483:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300. 00000100:00000010:3.0:1788972013.847487:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788972013.847491:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a600. 02000000:00000010:3.0:1788972013.847493:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972013.847495:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972013.847497:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:3.0:1788972013.847507:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788972013.847510:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800. 00000020:00000010:0.0:1788972014.846845:0:114674:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972014.846974:0:114674:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000100:00000010:2.0:1788972014.867583:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e240. 00000100:00000010:3.0:1788972014.872923:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e9c0. 00000020:00000010:3.0:1788972015.879636:0:114678:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972015.879732:0:114678:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:2.0:1788972016.905498:0:114682:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972016.906409:0:114682:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:0.0:1788972017.954561:0:114686:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972017.954652:0:114686:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000100:00000010:3.0:1788972018.962953:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800. 02000000:00000010:3.0:1788972018.962990:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788972018.963272:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972018.963279:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972018.963288:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788972018.963291:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300. 02000000:00000010:3.0:1788972018.963330:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ec00. 00000400:00000200:3.0:1788972018.963341:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972018.963345:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972018.963352:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788972018.963360:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579264, portal 10 00000400:00000010:3.0:1788972018.963368:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873110. 00000100:00000200:3.0:1788972018.963371:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189579264, offset 0 00000400:00000200:3.0:1788972018.963382:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972018.963392:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972018.963402:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972018.963408:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972018.963412:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972018.963417:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972018.963420:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972018.963424:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [516:516:256:4294967295] 192.168.203.150@tcp LPNI seq info [516:516:8:4294967295] 00000400:00000200:3.0:1788972018.963428:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972018.963439:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972018.963445:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972018.963452:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972018.963460:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972018.963485:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972018.963491:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972018.963505:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972018.963512:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09420. 00000100:00000010:3.0:1788972018.963535:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09ae0. 02000000:00000010:3.0:1788972018.963558:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f400. 00000400:00000200:3.0:1788972018.963562:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972018.963565:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972018.963569:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873bb0. 00000100:00000200:3.0:1788972018.963572:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579136, portal 25 00000400:00000010:3.0:1788972018.963575:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873c38. 00000100:00000200:3.0:1788972018.963578:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189579136, offset 0 00000400:00000200:3.0:1788972018.963582:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972018.963585:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972018.963588:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972018.963591:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972018.963594:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972018.963596:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972018.963598:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972018.963602:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [517:517:255:4294967295] 192.168.203.150@tcp LPNI seq info [517:517:7:4294967295] 00000400:00000200:3.0:1788972018.963605:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972018.963612:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972018.963616:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972018.963619:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788972018.963623:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972018.963626:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972018.963629:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972018.963633:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972018.963644:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ea00. 00000400:00000200:3.0:1788972018.963647:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972018.963654:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972018.963657:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972018.963659:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579392, portal 10 00000400:00000010:3.0:1788972018.963664:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0. 00000100:00000200:3.0:1788972018.963667:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189579392, offset 0 00000400:00000200:3.0:1788972018.963671:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972018.963674:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972018.963685:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972018.963688:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972018.963690:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972018.963692:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972018.963695:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972018.963698:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [518:518:254:4294967295] 192.168.203.150@tcp LPNI seq info [518:518:6:4294967295] 00000400:00000200:3.0:1788972018.963702:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972018.963708:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972018.963712:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972018.963716:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972018.963720:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972018.963724:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972018.963726:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972018.963731:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972018.964159:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972018.964170:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972018.964178:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972018.964184:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972018.964191:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873110 00000400:00000010:2.0:1788972018.964194:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873110. 00000100:00000200:2.0:1788972018.964207:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189579264/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972018.964232:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972018.964234:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:2.0:1788972018.964237:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972018.964240:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972018.964243:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873c38 00000400:00000010:2.0:1788972018.964244:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873c38. 00000100:00000200:2.0:1788972018.964249:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793800 x1875872189579136/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972018.964325:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972018.964328:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972018.964331:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972018.964334:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972018.964336:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0 00000400:00000010:2.0:1788972018.964338:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0. 00000100:00000200:2.0:1788972018.964343:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792300 x1875872189579392/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972018.966129:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966139:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972018.966146:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966151:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966156:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966160:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810000 00000400:00000200:2.0:1788972018.966166:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966172:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1945 [1] + 224 00000400:00000200:2.0:1788972018.966196:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972018.966201:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972018.966215:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189579264/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972018.966247:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966251:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972018.966254:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966257:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966260:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966262:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810080 00000400:00000200:2.0:1788972018.966266:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.966270:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1965 [1] + 224 00000400:00000200:2.0:1788972018.966276:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972018.966279:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972018.966286:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792300 x1875872189579392/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972018.966745:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972018.966752:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873880 00000400:00000010:3.0:1788972018.966754:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972018.966764:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792300 x1875872189579392/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972018.966773:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792300 x1875872189579392/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972018.966845:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972018.966849:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908 00000400:00000010:3.0:1788972018.966860:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788972018.966866:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189579264/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972018.966874:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189579264/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972018.966896:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ea00. 02000000:00000010:3.0:1788972018.966903:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300. 00000100:00000010:3.0:1788972018.966907:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788972018.966911:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ec00. 02000000:00000010:3.0:1788972018.966913:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972018.966915:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 00000400:00000200:2.0:1788972018.967330:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.967336:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972018.967342:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.967346:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.967349:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.967353:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780ff80 00000400:00000200:2.0:1788972018.967358:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972018.967364:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1955 [1] + 224 00000400:00000200:2.0:1788972018.967377:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972018.967381:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972018.967390:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793800 x1875872189579136/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972018.967636:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972018.967641:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873bb0 00000400:00000010:3.0:1788972018.967643:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873bb0. 00000100:00000200:3.0:1788972018.967648:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793800 x1875872189579136/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972018.967656:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793800 x1875872189579136/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972034 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972018.967685:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f400. 02000000:00000010:3.0:1788972018.967690:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788972018.967693:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800. 00000020:00000010:1.0:1788972019.014436:0:114691:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972019.014559:0:114691:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000100:00000010:2.0:1788972019.985992:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09420. 00000100:00000010:3.0:1788972019.986113:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09ae0. 00000020:00000010:3.0:1788972020.044067:0:114695:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972020.044171:0:114695:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:0.0:1788972021.074249:0:114699:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972021.074376:0:114699:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000020:00000010:3.0:1788972022.126446:0:114703:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972022.126566:0:114703:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:0.0:1788972023.154291:0:114707:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972023.154379:0:114707:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000100:00000010:1.0:1788972024.082101:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80. 02000000:00000010:1.0:1788972024.082121:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 00000100:00000010:1.0:1788972024.082348:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800. 02000000:00000010:1.0:1788972024.082357:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 00000100:00000010:1.0:1788972024.082376:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3b80. 02000000:00000010:1.0:1788972024.082382:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 02000000:00000010:0.0:1788972024.082510:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002800. 00000400:00000200:0.0:1788972024.082520:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972024.082523:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788972024.082529:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788972024.082535:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579520, portal 25 00000400:00000010:0.0:1788972024.082540:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8f68. 00000100:00000200:0.0:1788972024.082543:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189579520, offset 0 00000400:00000200:0.0:1788972024.082552:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972024.082560:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972024.082568:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972024.082573:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972024.082577:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972024.082581:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972024.082584:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972024.082588:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [519:519:256:4294967295] 192.168.203.150@tcp LPNI seq info [519:519:8:4294967295] 00000400:00000200:0.0:1788972024.082592:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972024.082602:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972024.082607:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972024.082613:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff600. 00000800:00000200:0.0:1788972024.082619:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972024.082627:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972024.082632:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972024.082653:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972024.082658:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d80. 00000100:00000010:0.0:1788972024.082672:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5420. 02000000:00000010:0.0:1788972024.082683:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002400. 00000400:00000200:0.0:1788972024.082686:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972024.082689:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972024.082706:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a85d8. 00000100:00000200:0.0:1788972024.082709:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579776, portal 10 00000400:00000010:0.0:1788972024.082715:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972024.082717:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189579776, offset 0 00000400:00000200:0.0:1788972024.082723:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972024.082726:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972024.082730:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972024.082732:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972024.082735:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972024.082737:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972024.082739:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972024.082742:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [520:520:255:4294967295] 192.168.203.150@tcp LPNI seq info [520:520:7:4294967295] 00000400:00000200:0.0:1788972024.082746:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972024.082752:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972024.082757:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972024.082760:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff300. 00000800:00000200:0.0:1788972024.082764:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972024.082768:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972024.082771:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972024.082777:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972024.082794:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002200. 00000400:00000200:0.0:1788972024.082797:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972024.082799:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972024.082802:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972024.082805:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579648, portal 10 00000400:00000010:0.0:1788972024.082809:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8110. 00000100:00000200:0.0:1788972024.082811:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189579648, offset 0 00000800:00000200:2.0:1788972024.082919:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972024.082923:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff600. 00000400:00000200:2.0:1788972024.082930:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972024.082935:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972024.082940:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8f68 00000400:00000010:2.0:1788972024.082942:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8f68. 00000100:00000200:2.0:1788972024.082951:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1f80 x1875872189579520/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972024.083027:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972024.083035:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff300. 00000400:00000200:2.0:1788972024.083038:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972024.083042:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972024.083044:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a87f8 00000400:00000010:2.0:1788972024.083046:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a87f8. 00000100:00000200:2.0:1788972024.083050:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3b80 x1875872189579776/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788972024.083707:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972024.083714:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972024.083718:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972024.083722:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972024.083733:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972024.083736:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972024.083739:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972024.083742:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [521:521:256:4294967295] 192.168.203.150@tcp LPNI seq info [521:521:8:4294967295] 00000400:00000200:0.0:1788972024.083747:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972024.083754:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972024.083759:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972024.083763:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffc00. 00000800:00000200:0.0:1788972024.083768:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972024.083783:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972024.083787:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972024.083803:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972024.083906:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972024.083909:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffc00. 00000400:00000200:2.0:1788972024.083913:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972024.083917:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972024.083921:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8110 00000400:00000010:2.0:1788972024.083922:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8110. 00000100:00000200:2.0:1788972024.083928:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3800 x1875872189579648/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972024.085133:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085606:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972024.085615:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085620:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085636:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085640:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810100 00000400:00000200:3.0:1788972024.085644:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085649:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1975 [1] + 224 00000400:00000200:3.0:1788972024.085661:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972024.085665:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972024.085677:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1f80 x1875872189579520/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972024.085704:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085707:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972024.085712:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085714:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085717:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085720:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810200 00000400:00000200:3.0:1788972024.085723:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972024.085727:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x197d [1] + 224 00000400:00000200:3.0:1788972024.085736:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972024.085739:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972024.085745:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3b80 x1875872189579776/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972024.088021:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788972024.088034:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:0.0:1788972024.088036:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788972024.088048:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1f80 x1875872189579520/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972024.088060:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1f80 x1875872189579520/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972024.088084:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972024.088087:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a85d8 00000400:00000010:0.0:1788972024.088089:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a85d8. 00000100:00000200:0.0:1788972024.088094:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3b80 x1875872189579776/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972024.088103:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3b80 x1875872189579776/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972024.088122:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002800. 02000000:00000010:0.0:1788972024.088129:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:0.0:1788972024.088134:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80. 02000000:00000010:0.0:1788972024.088155:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002400. 02000000:00000010:0.0:1788972024.088158:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:0.0:1788972024.088160:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3b80. 00000400:00000200:2.0:1788972024.092288:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972024.092298:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972024.092306:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972024.092311:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972024.092317:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972024.092321:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810180 00000400:00000200:2.0:1788972024.092326:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972024.092346:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1985 [1] + 224 00000400:00000200:2.0:1788972024.092362:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972024.092367:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972024.092382:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3800 x1875872189579648/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972024.092601:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972024.092607:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:0.0:1788972024.092609:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972024.092616:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3800 x1875872189579648/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972024.092625:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3800 x1875872189579648/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972040 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972024.092657:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002200. 02000000:00000010:0.0:1788972024.092664:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:0.0:1788972024.092668:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800. 00000020:00000010:3.0:1788972024.197458:0:114711:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972024.200256:0:114711:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:3.0:1788972025.107380:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5420. 00000100:00000010:2.0:1788972025.109287:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d80. 00000020:00000010:2.0:1788972025.234499:0:114715:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972025.234581:0:114715:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972026.264103:0:114720:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972026.264194:0:114720:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972027.288802:0:114724:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972027.288921:0:114724:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:3.0:1788972028.315850:0:114728:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972028.315943:0:114728:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:2.0:1788972029.202127:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972029.202148:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d500. 00000100:00000010:2.0:1788972029.202176:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972029.202181:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d200. 00000100:00000010:2.0:1788972029.202187:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972029.202190:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d100. 02000000:00000010:2.0:1788972029.202217:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094bc00. 00000400:00000200:2.0:1788972029.202226:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972029.202230:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01263c8. 00000400:00000010:2.0:1788972029.202236:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972029.202242:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580032, portal 10 00000400:00000010:2.0:1788972029.202249:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972029.202252:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189580032, offset 0 00000400:00000200:2.0:1788972029.202261:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972029.202270:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.202277:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972029.202282:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972029.202286:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972029.202289:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972029.202306:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972029.202310:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [522:522:256:4294967295] 192.168.203.150@tcp LPNI seq info [522:522:8:4294967295] 00000400:00000200:2.0:1788972029.202314:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.202323:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972029.202329:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972029.202336:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000800:00000200:2.0:1788972029.202343:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972029.202351:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972029.202356:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 02000000:00000010:3.0:1788972029.202366:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e000. 00000400:00000200:2.0:1788972029.202373:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972029.202376:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972029.202380:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e240. 00000400:00000010:3.0:1788972029.202381:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972029.202388:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873bb0. 00000100:00000200:3.0:1788972029.202392:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189579904, portal 25 00000100:00000010:2.0:1788972029.202393:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ec00. 00000400:00000010:3.0:1788972029.202405:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788972029.202408:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189579904, offset 0 00000400:00000200:3.0:1788972029.202416:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972029.202421:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972029.202426:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972029.202429:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972029.202444:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972029.202448:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972029.202462:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972029.202465:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [523:523:255:4294967295] 192.168.203.150@tcp LPNI seq info [523:523:7:4294967295] 00000400:00000200:3.0:1788972029.202470:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972029.202476:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972029.202480:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972029.202484:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00. 00000800:00000200:3.0:1788972029.202488:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972029.202494:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972029.202497:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972029.202502:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972029.202514:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f800. 00000400:00000200:3.0:1788972029.202518:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972029.202520:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972029.202523:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972029.202526:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580160, portal 10 00000400:00000010:3.0:1788972029.202531:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8. 00000100:00000200:3.0:1788972029.202533:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189580160, offset 0 00000400:00000200:3.0:1788972029.202538:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972029.202542:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972029.202545:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972029.202548:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972029.202550:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972029.202552:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972029.202561:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972029.202565:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [524:524:254:4294967295] 192.168.203.150@tcp LPNI seq info [524:524:6:4294967295] 00000400:00000200:3.0:1788972029.202569:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972029.202575:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972029.202580:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972029.202583:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972029.202587:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972029.202592:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972029.202594:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972029.202600:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972029.202710:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972029.202714:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000200:2.0:1788972029.202720:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972029.202725:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972029.202988:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0 00000400:00000010:2.0:1788972029.202991:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972029.203001:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189580032/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972029.203025:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972029.203028:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00. 00000400:00000200:2.0:1788972029.203031:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972029.203035:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972029.203037:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908 00000400:00000010:2.0:1788972029.203039:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908. 00000100:00000200:2.0:1788972029.203044:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189579904/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972029.203109:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972029.203112:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972029.203122:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972029.203125:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972029.203128:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8 00000400:00000010:2.0:1788972029.203130:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8. 00000100:00000200:2.0:1788972029.203135:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189580160/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972029.206472:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.206482:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972029.206490:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.206495:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.206500:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.206504:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810280 00000400:00000200:2.0:1788972029.206509:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.206515:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x199d [1] + 224 00000400:00000200:2.0:1788972029.206530:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972029.206535:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972029.206547:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189579904/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972029.206615:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972029.206620:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873bb0 00000400:00000010:3.0:1788972029.206623:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873bb0. 00000100:00000200:3.0:1788972029.206630:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189579904/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972029.206639:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189579904/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972029.206664:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e000. 02000000:00000010:3.0:1788972029.206669:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d500. 00000100:00000010:3.0:1788972029.206674:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 00000400:00000200:2.0:1788972029.207737:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207743:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972029.207749:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207752:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207756:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207759:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810380 00000400:00000200:2.0:1788972029.207763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207767:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x19ad [1] + 224 00000400:00000200:2.0:1788972029.207775:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972029.207778:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972029.207788:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189580160/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972029.207865:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207869:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972029.207872:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207875:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207877:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207880:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810300 00000400:00000200:2.0:1788972029.207883:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972029.207887:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x198d [1] + 224 00000400:00000200:2.0:1788972029.207895:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972029.207898:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972029.207904:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189580032/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972029.207935:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01263c8. 00000400:00000200:2.0:1788972029.207939:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:2.0:1788972029.207941:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972029.207947:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189580032/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972029.207956:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189580032/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972029.208046:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972029.208050:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873880 00000400:00000010:3.0:1788972029.208052:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972029.208059:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189580160/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972029.208070:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189580160/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972045 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972029.208087:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f800. 02000000:00000010:3.0:1788972029.208091:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d100. 00000100:00000010:3.0:1788972029.208102:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972029.208214:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094bc00. 02000000:00000010:2.0:1788972029.208223:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d200. 00000100:00000010:2.0:1788972029.208226:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 00000020:00000010:2.0:1788972029.362321:0:114732:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972029.362417:0:114732:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000100:00000010:3.0:1788972030.226177:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ec00. 00000100:00000010:2.0:1788972030.226560:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e240. 00000020:00000010:2.0:1788972030.396071:0:114736:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972030.398459:0:114736:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972031.422903:0:114740:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972031.422996:0:114740:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:3.0:1788972032.452100:0:114744:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972032.452194:0:114744:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972033.488540:0:114749:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972033.488630:0:114749:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:2.0:1788972034.322064:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972034.322087:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63900. 00000100:00000010:2.0:1788972034.322126:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840df80. 02000000:00000010:2.0:1788972034.322131:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63700. 00000100:00000010:2.0:1788972034.322138:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972034.322142:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63300. 02000000:00000010:3.0:1788972034.322473:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fa00. 00000400:00000200:3.0:1788972034.322483:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972034.322487:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972034.322493:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972034.322501:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580288, portal 25 00000400:00000010:3.0:1788972034.322506:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873bb0. 00000100:00000200:3.0:1788972034.322509:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189580288, offset 0 00000400:00000200:3.0:1788972034.322518:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972034.322526:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972034.322534:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972034.322539:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972034.322544:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972034.322547:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972034.322550:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972034.322554:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [525:525:256:4294967295] 192.168.203.150@tcp LPNI seq info [525:525:8:4294967295] 00000400:00000200:3.0:1788972034.322558:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972034.322566:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972034.322572:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972034.322577:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300. 00000800:00000200:3.0:1788972034.322583:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972034.322590:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972034.322594:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972034.322616:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972034.322623:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09ae0. 00000100:00000010:3.0:1788972034.322641:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09e40. 02000000:00000010:3.0:1788972034.322664:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e400. 00000400:00000200:3.0:1788972034.322668:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972034.322671:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972034.322674:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972034.322677:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580544, portal 10 00000400:00000010:3.0:1788972034.322681:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48736e8. 00000100:00000200:3.0:1788972034.322683:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189580544, offset 0 00000400:00000200:3.0:1788972034.322687:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972034.322691:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972034.322694:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972034.322696:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972034.322698:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972034.322700:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972034.322702:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972034.322705:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [526:526:255:4294967295] 192.168.203.150@tcp LPNI seq info [526:526:7:4294967295] 00000400:00000200:3.0:1788972034.322708:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972034.322713:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972034.322717:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972034.322720:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972034.322724:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972034.322727:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972034.322730:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972034.322735:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972034.322751:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788972034.322755:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972034.322757:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972034.322760:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8. 00000100:00000200:3.0:1788972034.322763:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580416, portal 10 00000400:00000010:3.0:1788972034.322767:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8. 00000100:00000200:3.0:1788972034.322769:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189580416, offset 0 00000400:00000200:3.0:1788972034.322773:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972034.322795:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972034.322798:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972034.322801:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972034.322803:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972034.322806:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972034.322808:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972034.322810:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [527:527:254:4294967295] 192.168.203.150@tcp LPNI seq info [527:527:6:4294967295] 00000400:00000200:3.0:1788972034.322843:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972034.322848:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972034.322853:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972034.322856:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00. 00000800:00000200:3.0:1788972034.322860:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972034.322864:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972034.322867:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972034.322872:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972034.323064:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972034.323068:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300. 00000400:00000200:2.0:1788972034.323075:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972034.323081:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972034.323086:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873bb0 00000400:00000010:2.0:1788972034.323088:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873bb0. 00000100:00000200:2.0:1788972034.323096:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189580288/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972034.323119:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972034.323121:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972034.323124:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972034.323127:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972034.323129:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48736e8 00000400:00000010:2.0:1788972034.323131:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48736e8. 00000100:00000200:2.0:1788972034.323135:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189580544/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972034.323205:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972034.323208:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00. 00000400:00000200:2.0:1788972034.323211:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972034.323214:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972034.323216:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8 00000400:00000010:2.0:1788972034.323217:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8. 00000100:00000200:2.0:1788972034.323222:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840df80 x1875872189580416/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972034.329167:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329191:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972034.329205:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329210:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329216:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329219:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810500 00000400:00000200:2.0:1788972034.329225:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329230:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x19cd [1] + 224 00000400:00000200:2.0:1788972034.329242:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972034.329247:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972034.329260:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189580544/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972034.329294:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329297:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972034.329302:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329304:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329307:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329310:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810480 00000400:00000200:2.0:1788972034.329313:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329317:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x19dd [1] + 224 00000400:00000200:2.0:1788972034.329323:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972034.329326:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972034.329332:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840df80 x1875872189580416/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972034.329353:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972034.329358:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972034.329360:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972034.329367:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189580544/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972034.329370:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329373:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000100:00000200:3.0:1788972034.329377:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189580544/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972034.329377:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329380:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329383:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329385:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810400 00000400:00000200:2.0:1788972034.329387:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972034.329396:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x19bd [1] + 224 00000400:00000200:2.0:1788972034.329404:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972034.329407:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 02000000:00000010:3.0:1788972034.329408:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e400. 00000100:00000200:2.0:1788972034.329412:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189580288/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972034.329423:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63300. 00000100:00000010:3.0:1788972034.329428:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 00000400:00000010:3.0:1788972034.329435:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972034.329437:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8 00000400:00000010:3.0:1788972034.329439:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8. 00000100:00000200:3.0:1788972034.329443:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840df80 x1875872189580416/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972034.329474:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840df80 x1875872189580416/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972034.329490:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972034.329493:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873880 00000400:00000010:3.0:1788972034.329495:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972034.329498:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189580288/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972034.329505:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189580288/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972050 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972034.329515:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:3.0:1788972034.329518:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63700. 00000100:00000010:3.0:1788972034.329520:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840df80. 02000000:00000010:3.0:1788972034.329523:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fa00. 02000000:00000010:3.0:1788972034.329525:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63900. 00000100:00000010:3.0:1788972034.329527:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 00000020:00000010:3.0:1788972034.521384:0:114753:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972034.521496:0:114753:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000100:00000010:2.0:1788972035.346348:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09ae0. 00000100:00000010:3.0:1788972035.346351:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09e40. 00000020:00000010:2.0:1788972035.561644:0:114757:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972035.561738:0:114757:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000020:00000010:0.0:1788972036.603145:0:114761:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972036.603239:0:114761:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000020:00000010:1.0:1788972037.657641:0:114765:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972037.657730:0:114765:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972038.683115:0:114769:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972038.683203:0:114769:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000100:00000010:0.0:1788972039.442450:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89180. 02000000:00000010:0.0:1788972039.442482:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff200. 00000100:00000010:0.0:1788972039.442516:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89880. 02000000:00000010:0.0:1788972039.442521:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 00000100:00000010:0.0:1788972039.442529:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89c00. 02000000:00000010:0.0:1788972039.442532:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cfff00. 02000000:00000010:0.0:1788972039.442558:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003e00. 00000400:00000200:0.0:1788972039.442567:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972039.442570:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972039.442576:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972039.442582:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580800, portal 10 00000400:00000010:0.0:1788972039.442588:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a85d8. 00000100:00000200:0.0:1788972039.442591:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189580800, offset 0 00000400:00000200:0.0:1788972039.442599:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972039.442607:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972039.442614:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972039.442630:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972039.442635:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972039.442639:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972039.442641:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972039.442645:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [528:528:256:4294967295] 192.168.203.150@tcp LPNI seq info [528:528:8:4294967295] 00000400:00000200:0.0:1788972039.442650:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972039.442659:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972039.442665:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972039.442670:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff800. 00000800:00000200:0.0:1788972039.442676:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972039.442684:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972039.442689:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972039.442708:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972039.442712:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c59c0. 02000000:00000010:1.0:1788972039.442810:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000400:00000200:1.0:1788972039.442830:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972039.442833:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972039.442837:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972039.442841:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580672, portal 25 00000400:00000010:1.0:1788972039.442844:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972039.442846:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189580672, offset 0 00000400:00000200:1.0:1788972039.442851:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972039.442855:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972039.442859:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972039.442862:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972039.442864:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972039.442866:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972039.442869:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972039.442872:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [529:529:255:4294967295] 192.168.203.150@tcp LPNI seq info [529:529:7:4294967295] 00000400:00000200:1.0:1788972039.442875:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972039.442882:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972039.442886:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972039.442895:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:1.0:1788972039.442899:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972039.442903:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972039.442906:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972039.442910:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972039.443867:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972039.443874:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff800. 00000400:00000200:2.0:1788972039.443880:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972039.443886:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972039.443892:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a85d8 00000400:00000010:2.0:1788972039.443894:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a85d8. 00000100:00000200:2.0:1788972039.443904:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89880 x1875872189580800/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972039.443997:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972039.444000:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:2.0:1788972039.444003:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972039.444006:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972039.444009:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:2.0:1788972039.444010:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:2.0:1788972039.444015:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89180 x1875872189580672/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1788972039.444275:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74673c0. 02000000:00000010:1.0:1788972039.444296:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788972039.444302:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972039.444306:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972039.444310:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972039.444314:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189580928, portal 10 00000400:00000010:1.0:1788972039.444320:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525908. 00000100:00000200:1.0:1788972039.444323:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189580928, offset 0 00000400:00000200:1.0:1788972039.444328:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972039.444334:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972039.444339:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972039.444342:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972039.444345:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972039.444348:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972039.444350:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972039.444354:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [530:530:256:4294967295] 192.168.203.150@tcp LPNI seq info [530:530:8:4294967295] 00000400:00000200:1.0:1788972039.444358:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972039.444365:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972039.444370:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972039.444374:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b100. 00000800:00000200:1.0:1788972039.444389:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972039.444396:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972039.444400:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972039.444419:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972039.445390:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972039.445396:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b100. 00000400:00000200:2.0:1788972039.445400:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972039.445407:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972039.445412:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525908 00000400:00000010:2.0:1788972039.445414:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525908. 00000100:00000200:2.0:1788972039.445421:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89c00 x1875872189580928/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972039.452645:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452654:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972039.452673:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452678:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452683:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452687:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810600 00000400:00000200:2.0:1788972039.452693:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452699:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x19ed [1] + 224 00000400:00000200:2.0:1788972039.452711:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972039.452716:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972039.452797:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89880 x1875872189580800/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972039.452853:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452857:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972039.452860:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452863:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452865:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452868:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810580 00000400:00000200:2.0:1788972039.452871:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452874:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x19f5 [1] + 224 00000400:00000200:2.0:1788972039.452880:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972039.452882:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972039.452888:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89180 x1875872189580672/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972039.452918:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452922:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972039.452926:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.452928:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.453046:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.453049:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810680 00000400:00000200:2.0:1788972039.453053:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972039.453057:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x19fd [1] + 224 00000400:00000200:2.0:1788972039.453068:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972039.453071:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972039.453077:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89c00 x1875872189580928/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972039.453171:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972039.453176:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:0.0:1788972039.453179:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972039.453185:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89880 x1875872189580800/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972039.453193:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89880 x1875872189580800/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972039.453308:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788972039.453313:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:1.0:1788972039.453320:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972039.453327:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89180 x1875872189580672/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972039.453335:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89180 x1875872189580672/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972039.453348:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788972039.453350:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:1.0:1788972039.453352:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972039.453356:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89c00 x1875872189580928/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972039.453363:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89c00 x1875872189580928/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972055 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972039.453387:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:1.0:1788972039.453394:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff200. 00000100:00000010:1.0:1788972039.453399:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89180. 02000000:00000010:1.0:1788972039.453403:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788972039.453405:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cfff00. 00000100:00000010:1.0:1788972039.453408:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89c00. 02000000:00000010:0.0:1788972039.453446:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003e00. 02000000:00000010:0.0:1788972039.453451:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:0.0:1788972039.453454:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89880. 00000020:00000010:1.0:1788972039.723994:0:114773:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972039.724084:0:114773:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000100:00000010:3.0:1788972040.466117:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74673c0. 00000100:00000010:2.0:1788972040.466735:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c59c0. 00000020:00000010:1.0:1788972040.762227:0:114778:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a0060. 00000020:00000010:1.0:1788972040.762317:0:114778:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a0060. 00000020:00000010:0.0:1788972041.813889:0:114782:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972041.813970:0:114782:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972042.845840:0:114786:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972042.845925:0:114786:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000020:00000010:2.0:1788972043.869601:0:114790:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4e730. 00000020:00000010:2.0:1788972043.869685:0:114790:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4e730. 00000100:00000010:0.0:1788972044.562037:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89880. 02000000:00000010:0.0:1788972044.562060:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 02000000:00000010:1.0:1788972044.562205:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788972044.562215:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972044.562219:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972044.562226:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972044.562232:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581056, portal 25 00000400:00000010:1.0:1788972044.562238:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972044.562241:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189581056, offset 0 00000400:00000200:1.0:1788972044.562251:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972044.562259:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972044.562273:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972044.562278:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972044.562282:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972044.562285:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972044.562288:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972044.562291:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [531:531:256:4294967295] 192.168.203.150@tcp LPNI seq info [531:531:8:4294967295] 00000400:00000200:1.0:1788972044.562296:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972044.562305:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972044.562312:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972044.562317:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:1.0:1788972044.562323:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972044.562331:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972044.562336:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972044.562368:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972044.562373:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467b40. 00000100:00000010:1.0:1788972044.562387:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467ba0. 00000800:00000200:2.0:1788972044.562519:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972044.562533:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:2.0:1788972044.562538:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972044.562543:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972044.562549:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:2.0:1788972044.562552:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:2.0:1788972044.562560:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89880 x1875872189581056/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1788972044.563356:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8aa00. 02000000:00000010:0.0:1788972044.563368:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 02000000:00000010:1.0:1788972044.563645:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0947a00. 00000400:00000200:1.0:1788972044.563654:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972044.563657:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972044.563662:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525d48. 00000100:00000200:1.0:1788972044.563666:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581184, portal 10 00000400:00000010:1.0:1788972044.563672:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75257f8. 00000100:00000200:1.0:1788972044.563675:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189581184, offset 0 00000400:00000200:1.0:1788972044.563681:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972044.563687:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972044.563692:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972044.563696:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972044.563699:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972044.563702:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972044.563705:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972044.563708:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [532:532:256:4294967295] 192.168.203.150@tcp LPNI seq info [532:532:8:4294967295] 00000400:00000200:2.0:1788972044.563973:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.563979:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972044.563985:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.563988:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.563994:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.563998:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810700 00000400:00000200:2.0:1788972044.564002:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.564007:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a05 [1] + 224 00000400:00000200:2.0:1788972044.564019:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972044.564024:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:1.0:1788972044.564033:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972044.564043:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000100:00000200:2.0:1788972044.564045:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89880 x1875872189581056/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1788972044.564048:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972044.564052:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972044.564059:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972044.564068:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972044.564072:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:0.0:1788972044.564150:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8bb80. 02000000:00000010:0.0:1788972044.564158:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff000. 02000000:00000010:0.0:1788972044.564229:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003e00. 00000400:00000200:0.0:1788972044.564234:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972044.564237:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972044.564241:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972044.564244:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581312, portal 10 00000400:00000010:0.0:1788972044.564247:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788972044.564249:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189581312, offset 0 00000400:00000200:0.0:1788972044.564254:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972044.564258:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972044.564262:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972044.564264:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972044.564266:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972044.564269:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972044.564271:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972044.564273:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [533:533:255:4294967295] 192.168.203.150@tcp LPNI seq info [533:533:7:4294967295] 00000400:00000200:0.0:1788972044.564277:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972044.564283:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972044.564286:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972044.564289:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff100. 00000800:00000200:0.0:1788972044.564293:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972044.564297:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972044.564299:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972044.564323:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972044.564332:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788972044.564336:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:1.0:1788972044.564338:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972044.564352:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89880 x1875872189581056/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972044.564362:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89880 x1875872189581056/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972044.564383:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788972044.564389:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:1.0:1788972044.564392:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89880. 00000800:00000200:2.0:1788972044.564926:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972044.564935:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:2.0:1788972044.564940:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972044.564945:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972044.564948:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75257f8 00000400:00000010:2.0:1788972044.564951:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75257f8. 00000100:00000200:2.0:1788972044.564957:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8aa00 x1875872189581184/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788972044.568931:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972044.569475:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972044.569481:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff100. 00000400:00000200:2.0:1788972044.569488:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972044.569495:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972044.569502:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:2.0:1788972044.569504:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:2.0:1788972044.569520:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8bb80 x1875872189581312/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972044.569619:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972044.569626:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972044.569633:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972044.569637:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972044.569642:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972044.569645:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810780 00000400:00000200:3.0:1788972044.569650:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972044.569656:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a0d [1] + 224 00000400:00000200:3.0:1788972044.569670:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972044.569685:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972044.569694:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8aa00 x1875872189581184/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972044.569800:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788972044.569806:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525d48 00000400:00000010:1.0:1788972044.569808:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525d48. 00000100:00000200:1.0:1788972044.569835:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8aa00 x1875872189581184/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972044.569844:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8aa00 x1875872189581184/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972044.570004:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0947a00. 02000000:00000010:1.0:1788972044.570011:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:1.0:1788972044.570016:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8aa00. 00000400:00000200:2.0:1788972044.575405:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.575414:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972044.575425:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.575430:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.575435:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.575831:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810800 00000400:00000200:2.0:1788972044.575838:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972044.575846:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a15 [1] + 224 00000400:00000200:2.0:1788972044.576150:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972044.576155:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972044.576170:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8bb80 x1875872189581312/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972044.576321:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972044.576327:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:0.0:1788972044.576329:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972044.576336:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8bb80 x1875872189581312/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972044.576369:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8bb80 x1875872189581312/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972060 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972044.576401:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003e00. 02000000:00000010:0.0:1788972044.576408:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff000. 00000100:00000010:0.0:1788972044.576412:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8bb80. 00000020:00000010:3.0:1788972044.903427:0:114794:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5140. 00000020:00000010:3.0:1788972044.903633:0:114794:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5140. 00000020:00000010:0.0:1788972044.931150:0:114795:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d47806b0. 00000020:00000010:0.0:1788972044.931240:0:114795:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d47806b0. 00000100:00000010:3.0:1788972045.586069:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467ba0. 00000100:00000010:2.0:1788972045.586164:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467b40. 00000100:00000010:2.0:1788972049.686442:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972049.686465:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972049.686500:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840df80. 02000000:00000010:2.0:1788972049.686504:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972049.686510:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972049.686513:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00. 02000000:00000010:3.0:1788972049.687395:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788972049.687406:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972049.687411:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972049.687418:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972049.687424:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581440, portal 25 00000400:00000010:3.0:1788972049.687429:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8. 00000100:00000200:3.0:1788972049.687432:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189581440, offset 0 00000400:00000200:3.0:1788972049.687440:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972049.687448:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972049.687455:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972049.687460:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972049.687464:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972049.687468:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972049.687471:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972049.687474:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [534:534:256:4294967295] 192.168.203.150@tcp LPNI seq info [534:534:8:4294967295] 00000400:00000200:3.0:1788972049.687478:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972049.687487:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972049.687493:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972049.687498:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972049.687504:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972049.687511:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972049.687515:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972049.687537:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972049.687542:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09a80. 00000100:00000010:3.0:1788972049.687566:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09e40. 02000000:00000010:3.0:1788972049.687578:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f400. 00000400:00000200:3.0:1788972049.687582:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972049.687585:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972049.687588:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972049.687591:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581696, portal 10 00000400:00000010:3.0:1788972049.687596:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908. 00000100:00000200:3.0:1788972049.687598:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189581696, offset 0 00000400:00000200:3.0:1788972049.687602:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972049.687606:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972049.687609:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972049.687611:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972049.687614:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972049.687616:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972049.687618:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972049.687621:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [535:535:255:4294967295] 192.168.203.150@tcp LPNI seq info [535:535:7:4294967295] 00000400:00000200:3.0:1788972049.687624:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972049.687630:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972049.687634:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972049.687637:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788972049.687640:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972049.687644:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972049.687647:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972049.687652:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972049.687669:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e000. 00000400:00000200:3.0:1788972049.687672:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972049.687674:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972049.687677:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0. 00000100:00000200:3.0:1788972049.687680:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581568, portal 10 00000400:00000010:3.0:1788972049.687684:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873c38. 00000100:00000200:3.0:1788972049.687686:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189581568, offset 0 00000400:00000200:3.0:1788972049.687696:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972049.687700:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972049.687703:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972049.687706:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972049.687709:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972049.687711:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972049.687713:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972049.687716:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [536:536:254:4294967295] 192.168.203.150@tcp LPNI seq info [536:536:6:4294967295] 00000400:00000200:3.0:1788972049.687720:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972049.687725:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972049.687728:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972049.687732:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972049.687735:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972049.687739:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972049.687741:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972049.687746:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972049.687903:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972049.687908:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972049.687914:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972049.687918:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972049.687923:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8 00000400:00000010:2.0:1788972049.687925:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8. 00000100:00000200:2.0:1788972049.687934:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189581440/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972049.687956:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972049.687958:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:2.0:1788972049.687960:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972049.687964:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972049.687966:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908 00000400:00000010:2.0:1788972049.687967:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908. 00000100:00000200:2.0:1788972049.687972:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189581696/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972049.688041:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972049.688044:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972049.688047:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972049.688051:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972049.688054:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873c38 00000400:00000010:2.0:1788972049.688055:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873c38. 00000100:00000200:2.0:1788972049.688060:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840df80 x1875872189581568/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972049.703264:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703285:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972049.703292:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703297:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703303:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703317:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810980 00000400:00000200:2.0:1788972049.703323:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703328:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a2d [1] + 224 00000400:00000200:2.0:1788972049.703339:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972049.703344:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972049.703356:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189581696/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972049.703388:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703392:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972049.703396:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703399:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703402:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703404:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810880 00000400:00000200:2.0:1788972049.703407:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703411:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a1d [1] + 224 00000400:00000200:2.0:1788972049.703416:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972049.703419:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972049.703425:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189581440/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972049.703437:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703440:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972049.703443:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703446:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703448:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703451:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810900 00000400:00000200:2.0:1788972049.703453:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972049.703461:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a3d [1] + 224 00000400:00000200:2.0:1788972049.703482:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972049.703485:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972049.703491:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840df80 x1875872189581568/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972049.703757:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972049.703763:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0 00000400:00000010:3.0:1788972049.703765:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0. 00000100:00000200:3.0:1788972049.703772:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840df80 x1875872189581568/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972049.703781:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840df80 x1875872189581568/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972049.703806:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972049.703808:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873880 00000400:00000010:3.0:1788972049.703810:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972049.703842:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189581440/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972049.703851:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189581440/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972049.703874:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972049.703877:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972049.703878:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972049.703882:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189581696/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972049.703891:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189581696/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972065 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972049.703905:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e000. 02000000:00000010:3.0:1788972049.703910:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:3.0:1788972049.703914:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840df80. 02000000:00000010:3.0:1788972049.703917:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:3.0:1788972049.703919:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:3.0:1788972049.703921:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 02000000:00000010:3.0:1788972049.703923:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f400. 02000000:00000010:3.0:1788972049.703925:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00. 00000100:00000010:3.0:1788972049.703927:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 00000100:00000010:2.0:1788972050.706016:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09a80. 00000100:00000010:3.0:1788972050.706055:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09e40. 00000100:00000010:2.0:1788972054.801962:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972054.801985:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 00000100:00000010:2.0:1788972054.802016:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972054.802020:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63500. 00000100:00000010:2.0:1788972054.802025:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972054.802028:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63100. 02000000:00000010:3.0:1788972054.802117:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788972054.802134:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972054.802140:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972054.802148:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972054.802157:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581824, portal 25 00000400:00000010:3.0:1788972054.802163:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873880. 00000100:00000200:3.0:1788972054.802167:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189581824, offset 0 00000400:00000200:3.0:1788972054.802178:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972054.802188:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972054.802200:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972054.802206:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972054.802211:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972054.802216:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972054.802239:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972054.802244:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [537:537:256:4294967295] 192.168.203.150@tcp LPNI seq info [537:537:8:4294967295] 00000400:00000200:3.0:1788972054.802249:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972054.802261:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972054.802282:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972054.802290:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972054.802299:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972054.802307:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972054.802313:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972054.802345:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972054.802353:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09e40. 00000100:00000010:3.0:1788972054.802372:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09ae0. 02000000:00000010:3.0:1788972054.802386:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e000. 00000400:00000200:3.0:1788972054.802390:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972054.802393:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972054.802397:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0. 00000100:00000200:3.0:1788972054.802400:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582080, portal 10 00000400:00000010:3.0:1788972054.802407:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873110. 00000100:00000200:3.0:1788972054.802409:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189582080, offset 0 00000400:00000200:3.0:1788972054.802414:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972054.802418:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972054.802421:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972054.802424:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972054.802426:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972054.802428:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972054.802431:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972054.802433:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [538:538:255:4294967295] 192.168.203.150@tcp LPNI seq info [538:538:7:4294967295] 00000400:00000200:3.0:1788972054.802437:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972054.802443:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972054.802447:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972054.802457:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788972054.802461:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972054.802466:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972054.802469:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972054.802475:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972054.802500:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ec00. 00000400:00000200:3.0:1788972054.802504:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972054.802507:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972054.802510:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788972054.802513:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189581952, portal 10 00000400:00000010:3.0:1788972054.802518:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873d48. 00000100:00000200:3.0:1788972054.802520:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189581952, offset 0 00000400:00000200:3.0:1788972054.802524:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972054.802528:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972054.802531:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972054.802534:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972054.802536:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972054.802538:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972054.802541:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972054.802543:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [539:539:254:4294967295] 192.168.203.150@tcp LPNI seq info [539:539:6:4294967295] 00000400:00000200:3.0:1788972054.802547:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972054.802553:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972054.802557:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972054.802561:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972054.802566:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972054.802570:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972054.802572:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972054.802578:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972054.802955:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972054.802963:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972054.802973:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972054.802982:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972054.802990:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873880 00000400:00000010:2.0:1788972054.802995:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873880. 00000100:00000200:2.0:1788972054.803012:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189581824/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972054.803043:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972054.803046:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:2.0:1788972054.803056:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972054.803061:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972054.803064:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873110 00000400:00000010:2.0:1788972054.803070:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873110. 00000100:00000200:2.0:1788972054.803076:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189582080/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972054.803119:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972054.803122:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972054.803125:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972054.803146:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972054.803151:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873d48 00000400:00000010:2.0:1788972054.803153:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873d48. 00000100:00000200:2.0:1788972054.803160:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189581952/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972054.818310:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818320:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972054.818331:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818335:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818340:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818344:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810b00 00000400:00000200:2.0:1788972054.818350:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818356:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a5d [1] + 224 00000400:00000200:2.0:1788972054.818368:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972054.818373:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972054.818387:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189582080/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972054.818946:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818951:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972054.818956:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818959:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818963:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818966:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810a00 00000400:00000200:2.0:1788972054.818970:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.818974:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a4d [1] + 224 00000400:00000200:2.0:1788972054.818981:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972054.818984:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972054.818991:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189581824/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972054.819026:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.819029:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972054.819033:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.819035:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.819038:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.819041:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810a80 00000400:00000200:2.0:1788972054.819044:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972054.819048:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a6d [1] + 224 00000400:00000200:2.0:1788972054.819058:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972054.819062:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972054.819068:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189581952/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972054.820914:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972054.820923:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0 00000400:00000010:3.0:1788972054.820926:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0. 00000100:00000200:3.0:1788972054.820951:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189582080/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972054.820962:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189582080/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972054.820995:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e000. 02000000:00000010:3.0:1788972054.821000:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63100. 00000100:00000010:3.0:1788972054.821005:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 00000400:00000010:3.0:1788972054.821013:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972054.821015:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:3.0:1788972054.821017:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788972054.821022:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189581952/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972054.821030:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189581952/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972054.821044:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972054.821046:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972054.821049:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972054.821053:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189581824/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972054.821062:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189581824/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972070 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972054.821072:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ec00. 02000000:00000010:3.0:1788972054.821075:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63500. 00000100:00000010:3.0:1788972054.821078:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 02000000:00000010:3.0:1788972054.821080:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:3.0:1788972054.821083:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:3.0:1788972054.821085:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 00000100:00000010:3.0:1788972055.827231:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09ae0. 00000100:00000010:2.0:1788972055.827426:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09e40. 00000100:00000010:3.0:1788972059.922104:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800. 02000000:00000010:3.0:1788972059.922154:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788972059.922198:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972059.922203:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972059.922212:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792300. 02000000:00000010:3.0:1788972059.922216:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67300. 02000000:00000010:2.0:1788972059.922257:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094aa00. 02000000:00000010:3.0:1788972059.922265:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e000. 00000400:00000200:2.0:1788972059.922274:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972059.922280:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972059.922285:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01263c8. 00000400:00000010:3.0:1788972059.922288:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:2.0:1788972059.922295:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000400:00000010:3.0:1788972059.922299:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:2.0:1788972059.922306:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582208, portal 25 00000100:00000200:3.0:1788972059.922308:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582336, portal 10 00000400:00000010:2.0:1788972059.922316:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0. 00000400:00000010:3.0:1788972059.922319:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:2.0:1788972059.922322:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189582208, offset 0 00000100:00000200:3.0:1788972059.922325:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189582336, offset 0 00000400:00000200:2.0:1788972059.922336:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972059.922356:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.922364:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972059.922368:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972059.922372:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972059.922377:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972059.922382:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972059.922388:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [540:540:256:4294967295] 192.168.203.150@tcp LPNI seq info [540:540:8:4294967295] 00000400:00000200:2.0:1788972059.922393:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.922415:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972059.922422:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972059.922428:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63200. 00000800:00000200:2.0:1788972059.922435:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972059.922441:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972059.922446:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972059.922461:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972059.922467:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e1e0. 00000100:00000010:2.0:1788972059.922477:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e240. 02000000:00000010:2.0:1788972059.922489:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788972059.922493:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972059.922496:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000200:3.0:1788972059.924928:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972059.924945:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972059.924956:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972059.924963:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972059.924968:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972059.924973:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972059.924976:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972059.924981:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [541:541:255:4294967295] 192.168.203.150@tcp LPNI seq info [541:541:7:4294967295] 00000400:00000200:3.0:1788972059.924986:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972059.924999:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972059.925006:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972059.925019:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000400:00000010:2.0:1788972059.925021:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38. 00000800:00000200:3.0:1788972059.925029:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972059.925039:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000100:00000200:2.0:1788972059.925040:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582464, portal 10 00000800:00000200:3.0:1788972059.925046:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:2.0:1788972059.925054:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972059.925060:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189582464, offset 0 00000400:00000200:3.0:1788972059.925070:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.925077:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972059.925090:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.925100:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972059.925107:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972059.925111:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972059.925116:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972059.925120:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972059.925124:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [542:542:254:4294967295] 192.168.203.150@tcp LPNI seq info [542:542:6:4294967295] 00000400:00000200:2.0:1788972059.925129:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.925139:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972059.925146:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972059.925153:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972059.925164:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972059.925172:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972059.925191:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972059.925199:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972059.925295:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972059.925299:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63200. 00000400:00000200:2.0:1788972059.925306:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972059.925310:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972059.925317:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0 00000400:00000010:2.0:1788972059.925319:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972059.925328:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793800 x1875872189582208/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972075 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972059.925351:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972059.925353:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972059.925357:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972059.925360:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972059.925362:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:2.0:1788972059.925364:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:2.0:1788972059.925369:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189582336/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972059.925440:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972059.925443:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:2.0:1788972059.925446:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972059.925452:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972059.925456:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440 00000400:00000010:2.0:1788972059.925459:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972059.925464:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792300 x1875872189582464/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972059.928636:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.928649:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972059.928660:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.928666:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.928672:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.928696:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810b80 00000400:00000200:2.0:1788972059.928703:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.928712:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a7d [1] + 224 00000400:00000200:2.0:1788972059.928730:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972059.928736:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972059.928756:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793800 x1875872189582208/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972059.928882:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01263c8. 00000400:00000200:3.0:1788972059.928889:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:3.0:1788972059.928892:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:3.0:1788972059.928907:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793800 x1875872189582208/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972059.928916:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793800 x1875872189582208/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972059.928952:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094aa00. 02000000:00000010:3.0:1788972059.928961:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788972059.928966:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800. 00000400:00000200:2.0:1788972059.933273:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933288:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972059.933303:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933309:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933315:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933320:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810c00 00000400:00000200:2.0:1788972059.933327:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933336:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a85 [1] + 224 00000400:00000200:2.0:1788972059.933351:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972059.933357:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972059.933380:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189582336/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972059.933483:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933487:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972059.933491:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933493:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933496:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933499:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810c80 00000400:00000200:2.0:1788972059.933502:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972059.933507:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1a9d [1] + 224 00000400:00000200:2.0:1788972059.933535:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972059.933539:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972059.933568:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792300 x1875872189582464/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972059.933635:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:2.0:1788972059.933640:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38 00000400:00000010:2.0:1788972059.933643:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38. 00000100:00000200:2.0:1788972059.933650:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792300 x1875872189582464/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972059.933659:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792300 x1875872189582464/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972059.933699:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:2.0:1788972059.933708:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67300. 00000100:00000010:2.0:1788972059.933715:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792300. 00000400:00000010:3.0:1788972059.933957:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972059.933962:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972059.933964:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972059.933971:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189582336/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972059.933982:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189582336/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972075 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972059.934000:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e000. 02000000:00000010:3.0:1788972059.934004:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972059.934007:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 00000100:00000010:3.0:1788972060.946533:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e240. 00000100:00000010:2.0:1788972060.946849:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e1e0. 00000100:00000010:0.0:1788972065.042545:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8bb80. 02000000:00000010:0.0:1788972065.042567:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffb00. 00000100:00000010:0.0:1788972065.042609:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a680. 02000000:00000010:0.0:1788972065.042614:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff400. 00000100:00000010:0.0:1788972065.042621:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8ad80. 02000000:00000010:0.0:1788972065.042624:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff900. 02000000:00000010:1.0:1788972065.042899:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788972065.042986:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972065.042992:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972065.043000:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525d48. 00000100:00000200:1.0:1788972065.043006:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582592, portal 25 00000400:00000010:1.0:1788972065.043011:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972065.043014:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189582592, offset 0 00000400:00000200:1.0:1788972065.043023:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972065.043031:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972065.043039:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972065.043043:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972065.043048:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972065.043051:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972065.043054:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972065.043058:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [543:543:256:4294967295] 192.168.203.150@tcp LPNI seq info [543:543:8:4294967295] 00000400:00000200:1.0:1788972065.043062:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972065.043073:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972065.043087:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972065.043092:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788972065.043099:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972065.043106:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972065.043111:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972065.043182:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972065.043188:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467d20. 00000100:00000010:1.0:1788972065.043207:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467c00. 02000000:00000010:1.0:1788972065.043219:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000400:00000200:1.0:1788972065.043222:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972065.043225:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972065.043228:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525aa0. 00000100:00000200:1.0:1788972065.043231:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582848, portal 10 00000400:00000010:1.0:1788972065.043235:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525bb0. 00000100:00000200:1.0:1788972065.043237:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189582848, offset 0 00000400:00000200:1.0:1788972065.043249:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972065.043253:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972065.043256:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972065.043258:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972065.043261:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972065.043263:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972065.043265:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972065.043267:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [544:544:255:4294967295] 192.168.203.150@tcp LPNI seq info [544:544:7:4294967295] 00000400:00000200:1.0:1788972065.043271:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972065.043277:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972065.043281:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972065.043284:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:1.0:1788972065.043288:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972065.043292:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972065.043295:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972065.043300:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972065.043318:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:1.0:1788972065.043321:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972065.043324:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788972065.043327:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75253b8. 00000100:00000200:1.0:1788972065.043330:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582720, portal 10 00000400:00000010:1.0:1788972065.043334:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75254c8. 00000100:00000200:1.0:1788972065.043336:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189582720, offset 0 00000400:00000200:1.0:1788972065.043340:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972065.043343:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972065.043346:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972065.043348:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972065.043350:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972065.043352:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972065.043355:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972065.043357:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [545:545:254:4294967295] 192.168.203.150@tcp LPNI seq info [545:545:6:4294967295] 00000400:00000200:1.0:1788972065.043377:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972065.043384:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972065.043388:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972065.043393:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972065.043396:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972065.043400:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972065.043402:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972065.043407:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972065.044705:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972065.044859:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:2.0:1788972065.044866:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972065.044873:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972065.044879:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:2.0:1788972065.044882:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:2.0:1788972065.044892:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8bb80 x1875872189582592/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972065.044979:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972065.044981:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:2.0:1788972065.044985:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972065.044989:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972065.044992:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525bb0 00000400:00000010:2.0:1788972065.044994:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525bb0. 00000100:00000200:2.0:1788972065.045000:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8ad80 x1875872189582848/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972065.045068:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972065.045081:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:2.0:1788972065.045084:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972065.045088:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972065.045091:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75254c8 00000400:00000010:2.0:1788972065.045093:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75254c8. 00000100:00000200:2.0:1788972065.045098:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a680 x1875872189582720/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972065.049022:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049035:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972065.049047:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049052:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049060:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049064:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810e00 00000400:00000200:2.0:1788972065.049071:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049079:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ab5 [1] + 224 00000400:00000200:2.0:1788972065.049093:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972065.049098:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972065.049116:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8ad80 x1875872189582848/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972065.049159:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049162:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972065.049166:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049169:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049171:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049174:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810d80 00000400:00000200:2.0:1788972065.049177:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049181:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1abd [1] + 224 00000400:00000200:2.0:1788972065.049186:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972065.049189:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972065.049206:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a680 x1875872189582720/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972065.049243:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049246:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972065.049250:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049253:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049256:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049258:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810d00 00000400:00000200:2.0:1788972065.049261:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972065.049266:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1aad [1] + 224 00000400:00000200:2.0:1788972065.049274:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972065.049277:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972065.049283:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8bb80 x1875872189582592/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972065.049646:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788972065.049654:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75253b8 00000400:00000010:1.0:1788972065.049657:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75253b8. 00000100:00000200:1.0:1788972065.049664:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a680 x1875872189582720/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972065.049674:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a680 x1875872189582720/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972065.049706:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788972065.049708:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525d48 00000400:00000010:1.0:1788972065.049711:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525d48. 00000100:00000200:1.0:1788972065.049714:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8bb80 x1875872189582592/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972065.049722:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8bb80 x1875872189582592/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972065.049732:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788972065.049734:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525aa0 00000400:00000010:1.0:1788972065.049736:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525aa0. 00000100:00000200:1.0:1788972065.049739:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8ad80 x1875872189582848/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972065.049746:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8ad80 x1875872189582848/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972081 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972065.049924:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:1.0:1788972065.049933:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff400. 00000100:00000010:1.0:1788972065.049938:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a680. 02000000:00000010:1.0:1788972065.049942:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788972065.049944:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffb00. 00000100:00000010:1.0:1788972065.049946:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8bb80. 02000000:00000010:1.0:1788972065.049949:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:1.0:1788972065.049951:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff900. 00000100:00000010:1.0:1788972065.049953:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8ad80. 00000100:00000010:3.0:1788972066.067223:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467c00. 00000100:00000010:2.0:1788972066.069170:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467d20. 00000100:00000010:3.0:1788972070.166012:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972070.166035:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972070.166076:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793800. 02000000:00000010:3.0:1788972070.166081:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67d00. 00000100:00000010:3.0:1788972070.166087:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100. 02000000:00000010:3.0:1788972070.166091:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 02000000:00000010:3.0:1788972070.166120:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f800. 00000400:00000200:3.0:1788972070.166131:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972070.166136:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972070.166143:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972070.166149:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189582976, portal 25 00000400:00000010:3.0:1788972070.166176:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0. 00000100:00000200:3.0:1788972070.166180:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189582976, offset 0 02000000:00000010:2.0:1788972070.166187:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:3.0:1788972070.166189:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972070.166192:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972070.166198:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972070.166198:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000010:2.0:1788972070.166202:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38. 00000400:00000200:3.0:1788972070.166205:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000100:00000200:2.0:1788972070.166205:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583104, portal 10 00000400:00000200:3.0:1788972070.166210:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000010:2.0:1788972070.166210:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972070.166211:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189583104, offset 0 00000400:00000200:3.0:1788972070.166215:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972070.166216:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972070.166218:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972070.166221:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972070.166225:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [546:546:256:4294967295] 192.168.203.150@tcp LPNI seq info [546:546:8:4294967295] 00000400:00000200:3.0:1788972070.166229:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972070.166239:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000400:00000200:2.0:1788972070.166244:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972070.166245:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972070.166247:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000800:00000010:3.0:1788972070.166249:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972070.166249:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972070.166252:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972070.166253:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000800:00000200:3.0:1788972070.166255:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:2.0:1788972070.166256:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972070.166258:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [547:547:255:4294967295] 192.168.203.150@tcp LPNI seq info [547:547:7:4294967295] 00000400:00000200:2.0:1788972070.166262:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972070.166263:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972070.166267:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972070.166273:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972070.166277:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972070.166279:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:2.0:1788972070.166280:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972070.166283:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000100:00000010:3.0:1788972070.166284:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09ae0. 00000800:00000200:2.0:1788972070.166287:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972070.166289:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:3.0:1788972070.166292:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09420. 00000400:00000200:2.0:1788972070.166295:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972070.166302:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fa00. 00000400:00000200:3.0:1788972070.166305:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972070.166307:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972070.166311:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972070.166313:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583232, portal 10 00000400:00000010:3.0:1788972070.166317:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873a18. 00000100:00000200:3.0:1788972070.166319:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189583232, offset 0 00000400:00000200:3.0:1788972070.166323:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972070.166326:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972070.166329:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972070.166332:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972070.166339:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972070.166341:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972070.166343:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972070.166346:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [548:548:254:4294967295] 192.168.203.150@tcp LPNI seq info [548:548:6:4294967295] 00000400:00000200:3.0:1788972070.166349:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972070.166354:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972070.166358:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000200:2.0:1788972070.166361:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972070.166362:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300. 00000800:00000010:2.0:1788972070.166365:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972070.166366:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972070.166369:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:2.0:1788972070.166371:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1788972070.166372:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972070.166376:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.166376:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972070.166381:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0 00000400:00000010:2.0:1788972070.166383:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0. 00000100:00000200:2.0:1788972070.166393:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189582976/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972070.166416:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972070.166418:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:2.0:1788972070.166421:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972070.166424:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972070.166426:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440 00000400:00000010:2.0:1788972070.166428:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972070.166432:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793800 x1875872189583104/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972070.166495:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972070.166502:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300. 00000400:00000200:2.0:1788972070.166505:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972070.166508:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972070.166511:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873a18 00000400:00000010:2.0:1788972070.166512:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873a18. 00000100:00000200:2.0:1788972070.166519:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793100 x1875872189583232/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972070.190419:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190429:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972070.190438:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190443:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190447:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190451:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887810e80 00000400:00000200:2.0:1788972070.190457:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190463:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ac5 [1] + 224 00000400:00000200:2.0:1788972070.190474:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972070.190479:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972070.190491:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189582976/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972070.190524:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190527:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972070.190532:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190534:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190549:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190552:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810f00 00000400:00000200:2.0:1788972070.190555:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190559:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ad5 [1] + 224 00000400:00000200:2.0:1788972070.190566:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972070.190569:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972070.190575:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793800 x1875872189583104/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972070.190597:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190601:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972070.190614:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190617:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190635:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190638:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887810f80 00000400:00000200:2.0:1788972070.190641:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972070.190644:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ae5 [1] + 224 00000400:00000200:2.0:1788972070.190654:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972070.190658:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972070.190664:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793100 x1875872189583232/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972070.190696:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:2.0:1788972070.190701:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38 00000400:00000010:2.0:1788972070.190704:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38. 00000100:00000200:2.0:1788972070.190710:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793800 x1875872189583104/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972070.190719:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793800 x1875872189583104/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972070.190749:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:2.0:1788972070.190757:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67d00. 00000100:00000010:2.0:1788972070.190761:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793800. 00000400:00000010:3.0:1788972070.190947:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972070.190952:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972070.190954:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972070.190960:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189582976/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972070.190969:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189582976/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972070.190982:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972070.190985:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:3.0:1788972070.190987:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972070.190991:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793100 x1875872189583232/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972070.190997:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793100 x1875872189583232/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972086 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972070.191009:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f800. 02000000:00000010:3.0:1788972070.191013:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972070.191016:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972070.191026:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fa00. 02000000:00000010:3.0:1788972070.191029:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788972070.191032:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100. 00000100:00000010:3.0:1788972071.186287:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09420. 00000100:00000010:2.0:1788972071.186379:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09ae0. 00000100:00000010:1.0:1788972075.282028:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972075.282050:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 00000100:00000010:1.0:1788972075.282084:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788972075.282098:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 00000100:00000010:1.0:1788972075.282105:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700. 02000000:00000010:1.0:1788972075.282108:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 02000000:00000010:1.0:1788972075.282239:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000400:00000200:1.0:1788972075.282249:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972075.282252:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972075.282258:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525aa0. 00000100:00000200:1.0:1788972075.282275:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583488, portal 10 00000400:00000010:1.0:1788972075.282282:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525d48. 00000100:00000200:1.0:1788972075.282285:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189583488, offset 0 00000400:00000200:1.0:1788972075.282294:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972075.282301:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972075.282308:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972075.282313:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972075.282317:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972075.282321:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972075.282323:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972075.282327:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [549:549:256:4294967295] 192.168.203.150@tcp LPNI seq info [549:549:8:4294967295] 00000400:00000200:1.0:1788972075.282345:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972075.282355:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972075.282361:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972075.282366:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972075.282372:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972075.282379:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972075.282383:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972075.282456:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972075.282462:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467600. 00000100:00000010:1.0:1788972075.282471:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74672a0. 02000000:00000010:1.0:1788972075.282499:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946200. 00000400:00000200:1.0:1788972075.282504:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972075.282507:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972075.282510:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75253b8. 00000100:00000200:1.0:1788972075.282513:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583360, portal 25 00000400:00000010:1.0:1788972075.282517:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525ee0. 00000100:00000200:1.0:1788972075.282519:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189583360, offset 0 00000400:00000200:1.0:1788972075.282524:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972075.282527:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972075.282531:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972075.282534:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972075.282536:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972075.282538:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972075.282540:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972075.282543:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [550:550:255:4294967295] 192.168.203.150@tcp LPNI seq info [550:550:7:4294967295] 00000400:00000200:1.0:1788972075.282547:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972075.282553:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972075.282557:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972075.282560:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:1.0:1788972075.282564:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972075.282573:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972075.282576:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972075.282580:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972075.282591:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:1.0:1788972075.282594:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972075.282597:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788972075.282600:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75255d8. 00000100:00000200:1.0:1788972075.282602:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583616, portal 10 00000400:00000010:1.0:1788972075.282607:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75254c8. 00000100:00000200:1.0:1788972075.282609:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189583616, offset 0 00000400:00000200:1.0:1788972075.282613:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972075.282616:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972075.282619:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972075.282622:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972075.282624:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972075.282626:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972075.282629:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972075.282632:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [551:551:254:4294967295] 192.168.203.150@tcp LPNI seq info [551:551:6:4294967295] 00000400:00000200:1.0:1788972075.282635:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972075.282652:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972075.282657:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972075.282659:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788972075.282663:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972075.282667:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972075.282669:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972075.282674:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972075.286404:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972075.286414:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:2.0:1788972075.286421:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972075.286430:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972075.286437:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525d48 00000400:00000010:2.0:1788972075.286439:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525d48. 00000100:00000200:2.0:1788972075.286452:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189583488/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972075.286476:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972075.286478:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:2.0:1788972075.286481:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972075.286496:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972075.286499:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525ee0 00000400:00000010:2.0:1788972075.286501:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525ee0. 00000100:00000200:2.0:1788972075.286506:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189583360/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972075.286570:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972075.286573:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:2.0:1788972075.286576:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972075.286580:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972075.286582:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75254c8 00000400:00000010:2.0:1788972075.286584:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75254c8. 00000100:00000200:2.0:1788972075.286589:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f0700 x1875872189583616/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972075.290928:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.290938:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972075.290945:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.290949:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.290954:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.290957:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811000 00000400:00000200:2.0:1788972075.290963:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.290967:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1afd [1] + 224 00000400:00000200:2.0:1788972075.290978:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972075.290982:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972075.291065:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189583360/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972075.291111:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291114:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972075.291119:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291121:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291133:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291150:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811080 00000400:00000200:2.0:1788972075.291155:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291159:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1af5 [1] + 224 00000400:00000200:2.0:1788972075.291168:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972075.291171:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972075.291178:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189583488/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972075.291192:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291195:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972075.291199:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291201:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291204:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291207:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811100 00000400:00000200:2.0:1788972075.291209:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972075.291212:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b05 [1] + 224 00000400:00000200:2.0:1788972075.291292:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972075.291297:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972075.291304:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f0700 x1875872189583616/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972075.292346:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:1.0:1788972075.292354:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75253b8 00000400:00000010:1.0:1788972075.292357:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75253b8. 00000100:00000200:1.0:1788972075.292366:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189583360/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972075.292393:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189583360/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972075.292428:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:1.0:1788972075.292432:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75255d8 00000400:00000010:1.0:1788972075.292434:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75255d8. 00000100:00000200:1.0:1788972075.292438:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f0700 x1875872189583616/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972075.292446:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f0700 x1875872189583616/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972075.292460:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:1.0:1788972075.292462:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525aa0 00000400:00000010:1.0:1788972075.292464:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525aa0. 00000100:00000200:1.0:1788972075.292468:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189583488/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972075.292475:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189583488/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972091 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972075.292490:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946200. 02000000:00000010:1.0:1788972075.292496:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:1.0:1788972075.292610:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972075.292617:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:1.0:1788972075.292621:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:1.0:1788972075.292624:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700. 02000000:00000010:1.0:1788972075.292626:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:1.0:1788972075.292628:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:1.0:1788972075.292631:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 00000100:00000010:2.0:1788972076.306049:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467600. 00000100:00000010:3.0:1788972076.306562:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74672a0. 00000100:00000010:0.0:1788972080.402655:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88380. 02000000:00000010:0.0:1788972080.402680:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff500. 00000100:00000010:0.0:1788972080.402708:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788972080.402712:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffe00. 00000100:00000010:0.0:1788972080.402718:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788972080.402723:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff900. 02000000:00000010:0.0:1788972080.402763:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d932c800. 00000400:00000200:0.0:1788972080.402772:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972080.402777:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972080.402784:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972080.402790:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583872, portal 10 00000400:00000010:0.0:1788972080.402795:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8bb0. 00000100:00000200:0.0:1788972080.402798:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189583872, offset 0 00000400:00000200:0.0:1788972080.402806:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972080.402831:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972080.402839:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972080.402859:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972080.402863:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972080.402867:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972080.402869:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972080.402873:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [552:552:256:4294967295] 192.168.203.150@tcp LPNI seq info [552:552:8:4294967295] 00000400:00000200:0.0:1788972080.402877:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972080.402887:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972080.402893:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972080.402898:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffb00. 00000800:00000200:0.0:1788972080.402905:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972080.402912:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972080.402917:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972080.402938:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972080.402943:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c50c0. 00000100:00000010:0.0:1788972080.402957:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5f60. 02000000:00000010:0.0:1788972080.402985:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c12e6600. 00000400:00000200:0.0:1788972080.402989:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972080.402991:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972080.402995:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972080.402998:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189583744, portal 25 00000400:00000010:0.0:1788972080.403001:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8cc0. 00000100:00000200:0.0:1788972080.403003:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189583744, offset 0 00000400:00000200:0.0:1788972080.403007:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972080.403011:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972080.403014:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972080.403017:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972080.403019:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972080.403021:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972080.403023:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972080.403025:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [553:553:255:4294967295] 192.168.203.150@tcp LPNI seq info [553:553:7:4294967295] 00000400:00000200:0.0:1788972080.403030:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972080.403043:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972080.403047:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972080.403050:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff400. 00000800:00000200:0.0:1788972080.403054:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972080.403058:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972080.403060:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972080.403065:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972080.403077:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c9446200. 00000400:00000200:0.0:1788972080.403080:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972080.403082:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788972080.403086:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972080.403099:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584000, portal 10 00000400:00000010:0.0:1788972080.403105:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788972080.403107:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189584000, offset 0 00000400:00000200:0.0:1788972080.403111:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972080.403114:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972080.403117:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972080.403120:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972080.403122:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972080.403125:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972080.403127:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972080.403129:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [554:554:254:4294967295] 192.168.203.150@tcp LPNI seq info [554:554:6:4294967295] 00000400:00000200:0.0:1788972080.403132:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972080.403137:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972080.403141:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972080.403144:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff000. 00000800:00000200:0.0:1788972080.403147:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972080.403151:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972080.403154:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972080.403158:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972080.403331:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972080.403336:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffb00. 00000400:00000200:2.0:1788972080.403342:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972080.403347:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972080.403352:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8bb0 00000400:00000010:2.0:1788972080.403355:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8bb0. 00000100:00000200:2.0:1788972080.403363:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a300 x1875872189583872/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972080.403384:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972080.403392:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff400. 00000400:00000200:2.0:1788972080.403395:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972080.403398:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972080.403401:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8cc0 00000400:00000010:2.0:1788972080.403403:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8cc0. 00000100:00000200:2.0:1788972080.403407:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88380 x1875872189583744/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972080.403478:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972080.403481:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff000. 00000400:00000200:2.0:1788972080.403484:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972080.403487:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972080.403490:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:2.0:1788972080.403491:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:2.0:1788972080.403496:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189584000/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972080.405557:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405565:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972080.405573:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405577:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405581:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405585:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811200 00000400:00000200:2.0:1788972080.405590:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405596:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b0d [1] + 224 00000400:00000200:2.0:1788972080.405606:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972080.405609:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972080.405622:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a300 x1875872189583872/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972080.405657:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405661:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972080.405665:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405680:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405683:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405686:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811280 00000400:00000200:2.0:1788972080.405689:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.405693:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b1d [1] + 224 00000400:00000200:2.0:1788972080.405702:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972080.405706:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972080.405713:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189584000/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972080.405960:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788972080.405966:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880 00000400:00000010:0.0:1788972080.405968:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972080.405975:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189584000/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972080.405984:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189584000/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972080.406012:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972080.406014:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:0.0:1788972080.406016:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972080.406019:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a300 x1875872189583872/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972080.406025:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a300 x1875872189583872/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972080.406040:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c9446200. 02000000:00000010:0.0:1788972080.406046:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff900. 00000100:00000010:0.0:1788972080.406051:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788972080.406054:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d932c800. 02000000:00000010:0.0:1788972080.406058:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffe00. 00000100:00000010:0.0:1788972080.406060:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a300. 00000400:00000200:2.0:1788972080.409778:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.409788:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972080.409796:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.409801:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.409807:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.409811:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811180 00000400:00000200:2.0:1788972080.409849:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972080.409855:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b15 [1] + 224 00000400:00000200:2.0:1788972080.409886:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972080.409891:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972080.409905:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88380 x1875872189583744/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972080.410482:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972080.410488:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088 00000400:00000010:0.0:1788972080.410491:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972080.410499:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88380 x1875872189583744/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972080.410510:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88380 x1875872189583744/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972096 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972080.410538:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c12e6600. 02000000:00000010:0.0:1788972080.410545:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff500. 00000100:00000010:0.0:1788972080.410550:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88380. 00000100:00000010:2.0:1788972081.426449:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c50c0. 00000100:00000010:3.0:1788972081.427218:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5f60. 00000100:00000010:2.0:1788972085.522269:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c380. 02000000:00000010:2.0:1788972085.522288:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 00000100:00000010:2.0:1788972085.522314:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ea00. 02000000:00000010:2.0:1788972085.522318:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63200. 00000100:00000010:2.0:1788972085.522323:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100. 02000000:00000010:2.0:1788972085.522326:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63500. 02000000:00000010:2.0:1788972085.522352:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:2.0:1788972085.522421:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972085.522425:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000010:2.0:1788972085.522432:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38. 00000100:00000200:2.0:1788972085.522439:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584128, portal 25 00000400:00000010:2.0:1788972085.522443:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972085.522446:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189584128, offset 0 00000400:00000200:2.0:1788972085.522454:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972085.522461:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.522468:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972085.522473:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972085.522476:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972085.522479:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972085.522482:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972085.522485:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [555:555:256:4294967295] 192.168.203.150@tcp LPNI seq info [555:555:8:4294967295] 00000400:00000200:2.0:1788972085.522488:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.522497:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972085.522502:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972085.522507:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63100. 00000800:00000200:2.0:1788972085.522512:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972085.522519:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972085.522523:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972085.522537:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972085.522541:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ec00. 00000100:00000010:2.0:1788972085.522550:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e1e0. 02000000:00000010:2.0:1788972085.522562:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094aa00. 00000400:00000200:2.0:1788972085.522566:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972085.522569:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f78. 00000400:00000010:2.0:1788972085.522572:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972085.522575:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584384, portal 10 00000400:00000010:2.0:1788972085.522579:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198. 00000100:00000200:2.0:1788972085.522581:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189584384, offset 0 00000400:00000200:2.0:1788972085.522593:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972085.522597:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.522601:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972085.522603:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972085.522605:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972085.522607:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972085.522609:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972085.522612:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [556:556:255:4294967295] 192.168.203.150@tcp LPNI seq info [556:556:7:4294967295] 00000400:00000200:2.0:1788972085.522615:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.522620:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972085.522624:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972085.522627:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00. 00000800:00000200:2.0:1788972085.522631:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972085.522636:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972085.522638:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972085.522644:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972085.522676:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a600. 00000400:00000200:2.0:1788972085.522680:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972085.522683:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972085.522686:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972085.522689:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584256, portal 10 00000400:00000010:2.0:1788972085.522692:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b770. 00000100:00000200:2.0:1788972085.522694:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189584256, offset 0 00000400:00000200:2.0:1788972085.522699:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972085.522702:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.522705:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972085.522708:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972085.522710:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972085.522712:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972085.522714:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972085.522717:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [557:557:254:4294967295] 192.168.203.150@tcp LPNI seq info [557:557:6:4294967295] 00000400:00000200:2.0:1788972085.522727:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.522734:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972085.522738:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972085.522742:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63d00. 00000800:00000200:2.0:1788972085.522745:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972085.522749:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972085.522752:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972085.522757:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972085.522798:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972085.522801:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63100. 00000400:00000200:2.0:1788972085.522807:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972085.522811:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972085.522834:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440 00000400:00000010:2.0:1788972085.522836:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972085.522846:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c380 x1875872189584128/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972101 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972085.522867:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972085.522869:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00. 00000400:00000200:2.0:1788972085.522872:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972085.522875:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972085.522877:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198 00000400:00000010:2.0:1788972085.522879:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198. 00000100:00000200:2.0:1788972085.522883:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f100 x1875872189584384/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972085.522949:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972085.522951:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63d00. 00000400:00000200:2.0:1788972085.522954:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972085.522957:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972085.522960:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b770 00000400:00000010:2.0:1788972085.522961:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b770. 00000100:00000200:2.0:1788972085.522966:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ea00 x1875872189584256/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972085.528573:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.528583:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972085.528591:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.528596:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.528602:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.528605:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811400 00000400:00000200:2.0:1788972085.528611:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.528617:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b35 [1] + 224 00000400:00000200:2.0:1788972085.528630:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972085.528635:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972085.528648:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f100 x1875872189584384/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972085.528749:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f78. 00000400:00000200:3.0:1788972085.528754:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0 00000400:00000010:3.0:1788972085.528756:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0. 00000100:00000200:3.0:1788972085.528763:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f100 x1875872189584384/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972085.528772:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f100 x1875872189584384/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972085.528801:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094aa00. 02000000:00000010:3.0:1788972085.528808:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63500. 00000100:00000010:3.0:1788972085.528841:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100. 00000400:00000200:2.0:1788972085.531385:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.531394:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972085.531403:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.531408:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.531413:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.531417:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811300 00000400:00000200:2.0:1788972085.531422:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.531429:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b25 [1] + 224 00000400:00000200:2.0:1788972085.531443:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972085.531447:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972085.531459:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c380 x1875872189584128/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972085.534922:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:3.0:1788972085.534935:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38 00000400:00000010:3.0:1788972085.534938:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38. 00000100:00000200:3.0:1788972085.534951:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c380 x1875872189584128/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972085.534963:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c380 x1875872189584128/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972085.534991:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:3.0:1788972085.535724:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:3.0:1788972085.535735:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c380. 00000400:00000200:2.0:1788972085.538659:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.538669:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972085.538678:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.538682:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.538701:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.538705:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811380 00000400:00000200:2.0:1788972085.538711:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972085.538716:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b45 [1] + 224 00000400:00000200:2.0:1788972085.538730:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972085.538734:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972085.538748:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ea00 x1875872189584256/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972085.538909:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:3.0:1788972085.538914:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0 00000400:00000010:3.0:1788972085.538917:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0. 00000100:00000200:3.0:1788972085.538924:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ea00 x1875872189584256/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972085.538944:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ea00 x1875872189584256/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972101 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972085.538976:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a600. 02000000:00000010:3.0:1788972085.538983:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63200. 00000100:00000010:3.0:1788972085.538987:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ea00. 00000100:00000010:3.0:1788972086.545953:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e1e0. 00000100:00000010:2.0:1788972086.546040:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ec00. 00000100:00000010:2.0:1788972090.645683:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e300. 02000000:00000010:2.0:1788972090.645702:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972090.645740:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972090.645745:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00. 00000100:00000010:2.0:1788972090.645752:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972090.645757:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63100. 02000000:00000010:3.0:1788972090.645893:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fc00. 00000400:00000200:3.0:1788972090.645984:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972090.645990:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972090.645997:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972090.646003:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584512, portal 25 00000400:00000010:3.0:1788972090.646009:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972090.646012:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189584512, offset 0 00000400:00000200:3.0:1788972090.646031:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972090.646041:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972090.646049:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972090.646054:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972090.646059:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972090.646063:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972090.646065:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972090.646068:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [558:558:256:4294967295] 192.168.203.150@tcp LPNI seq info [558:558:8:4294967295] 00000400:00000200:3.0:1788972090.646073:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972090.646081:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972090.646086:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972090.646092:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67600. 00000800:00000200:3.0:1788972090.646098:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972090.646153:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972090.646158:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972090.646170:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972090.646182:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09420. 00000100:00000010:3.0:1788972090.646191:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09300. 02000000:00000010:3.0:1788972090.646203:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fa00. 00000400:00000200:3.0:1788972090.646208:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972090.646211:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972090.646215:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972090.646219:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584768, portal 10 00000400:00000010:3.0:1788972090.646224:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873660. 00000100:00000200:3.0:1788972090.646226:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189584768, offset 0 00000400:00000200:3.0:1788972090.646232:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972090.646238:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972090.646242:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972090.646245:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972090.646248:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972090.646251:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972090.646254:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972090.646257:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [559:559:255:4294967295] 192.168.203.150@tcp LPNI seq info [559:559:7:4294967295] 00000400:00000200:3.0:1788972090.646262:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972090.646267:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972090.646272:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972090.646276:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972090.646280:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972090.646285:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972090.646288:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972090.646295:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972090.646944:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f800. 00000400:00000200:3.0:1788972090.646951:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972090.646954:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972090.646958:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788972090.646961:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584640, portal 10 00000400:00000010:3.0:1788972090.646966:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788972090.646976:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189584640, offset 0 00000400:00000200:3.0:1788972090.646981:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972090.646985:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972090.646989:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972090.646992:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972090.646995:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972090.646997:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972090.646999:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972090.647002:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [560:560:254:4294967295] 192.168.203.150@tcp LPNI seq info [560:560:6:4294967295] 00000400:00000200:3.0:1788972090.647006:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972090.647012:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972090.647027:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972090.647031:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972090.647035:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972090.647040:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972090.647043:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972090.647048:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972090.647163:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972090.647167:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67600. 00000400:00000200:2.0:1788972090.647172:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972090.647177:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972090.647182:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:2.0:1788972090.647185:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:2.0:1788972090.647193:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e300 x1875872189584512/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972106 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972090.647212:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972090.647214:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972090.647217:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972090.647220:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972090.647222:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873660 00000400:00000010:2.0:1788972090.647224:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873660. 00000100:00000200:2.0:1788972090.647229:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189584768/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972090.647295:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972090.648874:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972090.648888:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972090.648895:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972090.648902:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770 00000400:00000010:2.0:1788972090.648904:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770. 00000100:00000200:2.0:1788972090.648912:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189584640/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972090.656792:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656803:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972090.656811:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656852:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656858:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656866:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811580 00000400:00000200:2.0:1788972090.656872:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656877:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b65 [1] + 224 00000400:00000200:2.0:1788972090.656891:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972090.656897:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972090.656912:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189584768/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972090.656957:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656961:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972090.656967:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656970:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656973:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656975:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811500 00000400:00000200:2.0:1788972090.656980:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.656984:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b75 [1] + 224 00000400:00000200:2.0:1788972090.657010:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972090.657037:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000010:3.0:1788972090.657044:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972090.657056:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972090.657059:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972090.657073:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189584768/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972090.657085:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189584768/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972090.657125:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fa00. 02000000:00000010:3.0:1788972090.657133:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63100. 00000100:00000010:3.0:1788972090.657139:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000100:00000200:2.0:1788972090.657801:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189584640/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972090.661075:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972090.661090:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28 00000400:00000010:3.0:1788972090.661093:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28. 00000400:00000200:2.0:1788972090.661159:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:3.0:1788972090.661176:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189584640/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972090.661176:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972090.661194:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:3.0:1788972090.661197:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189584640/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972090.661202:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.661208:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.661213:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811480 00000400:00000200:2.0:1788972090.661218:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972090.661224:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b55 [1] + 224 00000400:00000200:2.0:1788972090.661241:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972090.661246:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972090.661258:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e300 x1875872189584512/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972090.662631:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972090.662637:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:3.0:1788972090.662639:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972090.662650:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e300 x1875872189584512/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972090.662661:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e300 x1875872189584512/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972106 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972090.662678:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f800. 02000000:00000010:3.0:1788972090.662686:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00. 00000100:00000010:3.0:1788972090.662691:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 02000000:00000010:3.0:1788972090.662696:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fc00. 02000000:00000010:3.0:1788972090.662699:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:3.0:1788972090.662701:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e300. 00000100:00000010:3.0:1788972091.665933:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09300. 00000100:00000010:2.0:1788972091.665946:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09420. 00000100:00000010:2.0:1788972095.761988:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ce00. 02000000:00000010:2.0:1788972095.762013:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972095.762046:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f800. 02000000:00000010:2.0:1788972095.762050:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63900. 00000100:00000010:2.0:1788972095.762055:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788972095.762058:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63700. 02000000:00000010:2.0:1788972095.762091:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094bc00. 00000400:00000200:2.0:1788972095.762103:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972095.762108:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126318. 00000400:00000010:2.0:1788972095.762115:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b770. 00000100:00000200:2.0:1788972095.762124:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189584896, portal 25 00000400:00000010:2.0:1788972095.762129:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198. 00000100:00000200:2.0:1788972095.762131:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189584896, offset 0 00000400:00000200:2.0:1788972095.762139:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972095.762160:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.762167:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972095.762171:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972095.762175:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972095.762178:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972095.762181:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972095.762185:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [561:561:256:4294967295] 192.168.203.150@tcp LPNI seq info [561:561:8:4294967295] 00000400:00000200:2.0:1788972095.762188:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.762197:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972095.762203:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972095.762207:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63300. 00000800:00000200:2.0:1788972095.762214:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972095.762222:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972095.762227:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972095.762245:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972095.762251:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e9c0. 00000100:00000010:2.0:1788972095.762275:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e120. 02000000:00000010:2.0:1788972095.762287:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a400. 00000400:00000200:2.0:1788972095.762290:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972095.762292:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126e70. 00000400:00000010:2.0:1788972095.762296:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440. 00000100:00000200:2.0:1788972095.762298:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585152, portal 10 00000400:00000010:2.0:1788972095.762305:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972095.762307:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189585152, offset 0 00000400:00000200:2.0:1788972095.762310:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972095.762314:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.762317:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972095.762319:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972095.762321:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972095.762323:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972095.762325:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972095.762333:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [562:562:255:4294967295] 192.168.203.150@tcp LPNI seq info [562:562:7:4294967295] 00000400:00000200:2.0:1788972095.762336:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.762341:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972095.762345:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972095.762347:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63d00. 00000800:00000200:2.0:1788972095.762350:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972095.762354:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972095.762356:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972095.762362:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972095.762387:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:2.0:1788972095.762391:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972095.762394:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580. 00000400:00000010:2.0:1788972095.762397:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38. 00000100:00000200:2.0:1788972095.762400:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585024, portal 10 00000400:00000010:2.0:1788972095.762404:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972095.762406:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189585024, offset 0 00000400:00000200:2.0:1788972095.762411:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972095.762414:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.762418:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972095.762421:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972095.762423:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972095.762425:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972095.762427:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972095.762430:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [563:563:254:4294967295] 192.168.203.150@tcp LPNI seq info [563:563:6:4294967295] 00000400:00000200:2.0:1788972095.762434:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.762439:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972095.762443:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972095.762446:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00. 00000800:00000200:2.0:1788972095.762449:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972095.762453:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972095.762461:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972095.762467:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972095.762514:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972095.762517:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63300. 00000400:00000200:2.0:1788972095.762523:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972095.762527:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972095.762532:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198 00000400:00000010:2.0:1788972095.762534:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198. 00000100:00000200:2.0:1788972095.762543:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ce00 x1875872189584896/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972111 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972095.762577:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972095.762579:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63d00. 00000400:00000200:2.0:1788972095.762582:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972095.762585:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972095.762587:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0 00000400:00000010:2.0:1788972095.762589:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972095.762593:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189585152/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972095.762653:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972095.762656:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00. 00000400:00000200:2.0:1788972095.762658:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972095.762661:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972095.762663:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0 00000400:00000010:2.0:1788972095.762665:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972095.762669:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f800 x1875872189585024/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972095.766390:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766399:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972095.766407:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766412:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766417:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766435:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811700 00000400:00000200:2.0:1788972095.766440:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766447:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b95 [1] + 224 00000400:00000200:2.0:1788972095.766459:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972095.766463:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972095.766477:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189585152/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972095.766517:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766521:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972095.766525:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766527:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766530:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766533:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811680 00000400:00000010:3.0:1788972095.766578:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126e70. 00000400:00000200:3.0:1788972095.766584:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440 00000400:00000010:3.0:1788972095.766586:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440. 00000100:00000200:3.0:1788972095.766593:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189585152/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972095.766604:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189585152/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972095.766611:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.766615:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ba5 [1] + 224 00000400:00000200:2.0:1788972095.766625:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972095.766628:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972095.766634:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f800 x1875872189585024/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972095.766636:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a400. 02000000:00000010:3.0:1788972095.766642:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63700. 00000100:00000010:3.0:1788972095.766647:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000400:00000010:3.0:1788972095.766653:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580. 00000400:00000200:3.0:1788972095.766662:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38 00000400:00000010:3.0:1788972095.766664:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38. 00000100:00000200:3.0:1788972095.766669:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f800 x1875872189585024/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972095.766677:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f800 x1875872189585024/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972095.766693:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:3.0:1788972095.766697:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63900. 00000100:00000010:3.0:1788972095.766700:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f800. 00000400:00000200:2.0:1788972095.767924:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.767931:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972095.767938:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.767941:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.767945:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.767948:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811600 00000400:00000200:2.0:1788972095.767953:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972095.767958:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1b85 [1] + 224 00000400:00000200:2.0:1788972095.767970:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972095.767974:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972095.768006:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ce00 x1875872189584896/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972095.768168:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126318. 00000400:00000200:3.0:1788972095.768173:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b770 00000400:00000010:3.0:1788972095.768175:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b770. 00000100:00000200:3.0:1788972095.768181:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ce00 x1875872189584896/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972095.768190:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ce00 x1875872189584896/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972111 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972095.768209:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094bc00. 02000000:00000010:3.0:1788972095.768215:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:3.0:1788972095.768218:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ce00. 00000100:00000010:2.0:1788972096.786488:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e9c0. 00000100:00000010:3.0:1788972096.787138:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e120. 00000100:00000010:2.0:1788972100.887645:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80. 02000000:00000010:2.0:1788972100.887999:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00. 00000100:00000010:2.0:1788972100.888043:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480. 02000000:00000010:2.0:1788972100.888049:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972100.888058:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ce00. 02000000:00000010:2.0:1788972100.888061:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63300. 02000000:00000010:2.0:1788972100.888097:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b600. 00000400:00000200:2.0:1788972100.888110:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972100.888118:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126a50. 00000400:00000010:2.0:1788972100.888127:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0. 00000100:00000200:2.0:1788972100.888137:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585280, portal 25 00000400:00000010:2.0:1788972100.888144:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972100.888147:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189585280, offset 0 00000400:00000200:2.0:1788972100.888158:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972100.888170:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.888183:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972100.888190:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972100.888195:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972100.888200:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972100.888203:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972100.888208:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [564:564:256:4294967295] 192.168.203.150@tcp LPNI seq info [564:564:8:4294967295] 00000400:00000200:2.0:1788972100.888213:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.888224:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972100.888235:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972100.888241:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63100. 00000800:00000200:2.0:1788972100.888249:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972100.888258:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972100.888298:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972100.888323:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972100.888331:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e9c0. 00000100:00000010:2.0:1788972100.888345:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e2a0. 02000000:00000010:2.0:1788972100.888358:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b800. 00000400:00000200:2.0:1788972100.888362:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972100.888365:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126948. 00000400:00000010:2.0:1788972100.888369:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198. 00000100:00000200:2.0:1788972100.888372:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585536, portal 10 00000400:00000010:2.0:1788972100.888379:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972100.888381:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189585536, offset 0 00000400:00000200:2.0:1788972100.888386:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972100.888390:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.888394:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972100.888397:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972100.888400:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972100.888402:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972100.888404:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972100.888407:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [565:565:255:4294967295] 192.168.203.150@tcp LPNI seq info [565:565:7:4294967295] 00000400:00000200:2.0:1788972100.888411:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.888416:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972100.888421:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972100.888424:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63200. 00000800:00000200:2.0:1788972100.888428:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972100.888432:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972100.888434:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972100.888441:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972100.888463:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094bc00. 00000400:00000200:2.0:1788972100.888467:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972100.888469:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f20. 00000400:00000010:2.0:1788972100.888472:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220. 00000100:00000200:2.0:1788972100.888485:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585408, portal 10 00000400:00000010:2.0:1788972100.888490:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972100.888492:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189585408, offset 0 00000400:00000200:2.0:1788972100.888497:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972100.888500:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.888503:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972100.888506:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972100.888508:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972100.888510:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972100.888512:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972100.888515:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [566:566:254:4294967295] 192.168.203.150@tcp LPNI seq info [566:566:6:4294967295] 00000400:00000200:2.0:1788972100.888519:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.888524:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972100.888528:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972100.888531:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972100.888534:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972100.888538:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972100.888541:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972100.888559:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972100.888797:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972100.888802:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63100. 00000400:00000200:2.0:1788972100.888808:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972100.888900:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972100.888908:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0 00000400:00000010:2.0:1788972100.888911:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972100.888923:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ca80 x1875872189585280/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972116 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972100.888960:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972100.888963:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63200. 00000400:00000200:2.0:1788972100.888965:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972100.888969:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972100.888978:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:2.0:1788972100.888980:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:2.0:1788972100.888986:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ce00 x1875872189585536/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972100.889072:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972100.889075:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:2.0:1788972100.889078:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972100.889081:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972100.889084:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68 00000400:00000010:2.0:1788972100.889086:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972100.889090:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f480 x1875872189585408/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972100.892955:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.892967:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972100.892978:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.892984:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.892991:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.892996:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811880 00000400:00000200:2.0:1788972100.893003:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893011:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1bc5 [1] + 224 00000400:00000200:2.0:1788972100.893026:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972100.893032:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972100.893049:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ce00 x1875872189585536/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972100.893087:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893091:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972100.893095:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893098:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893101:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893104:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811800 00000400:00000200:2.0:1788972100.893127:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893132:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1bd5 [1] + 224 00000400:00000010:3.0:1788972100.893170:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126948. 00000400:00000200:2.0:1788972100.893173:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972100.893176:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198 00000400:00000010:3.0:1788972100.893179:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198. 00000400:00000200:2.0:1788972100.893180:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972100.893188:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ce00 x1875872189585536/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972100.893192:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f480 x1875872189585408/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972100.893202:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ce00 x1875872189585536/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972100.893224:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893227:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972100.893230:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893233:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893236:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893238:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811780 00000400:00000200:2.0:1788972100.893242:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972100.893247:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1bb5 [1] + 224 00000400:00000200:2.0:1788972100.893254:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972100.893257:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972100.893264:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ca80 x1875872189585280/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972100.893360:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b800. 02000000:00000010:3.0:1788972100.893369:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63300. 00000100:00000010:3.0:1788972100.893375:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ce00. 00000400:00000010:3.0:1788972100.893391:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f20. 00000400:00000200:3.0:1788972100.893394:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220 00000400:00000010:3.0:1788972100.893396:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220. 00000100:00000200:3.0:1788972100.893402:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f480 x1875872189585408/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972100.893411:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f480 x1875872189585408/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972100.893427:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126a50. 00000400:00000200:3.0:1788972100.893430:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0 00000400:00000010:3.0:1788972100.893432:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0. 00000100:00000200:3.0:1788972100.893437:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ca80 x1875872189585280/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972100.893445:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ca80 x1875872189585280/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972116 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972100.893457:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094bc00. 02000000:00000010:3.0:1788972100.893461:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:3.0:1788972100.893463:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480. 02000000:00000010:3.0:1788972100.893467:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b600. 02000000:00000010:3.0:1788972100.893469:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00. 00000100:00000010:3.0:1788972100.893471:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80. 00000100:00000010:3.0:1788972101.906215:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e2a0. 00000100:00000010:2.0:1788972101.908211:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e9c0. 00000100:00000010:3.0:1788972106.002178:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100. 02000000:00000010:3.0:1788972106.002211:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788972106.002251:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972106.002257:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972106.002264:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000. 02000000:00000010:3.0:1788972106.002268:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 02000000:00000010:3.0:1788972106.002304:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f400. 00000400:00000200:3.0:1788972106.002316:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972106.002321:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972106.002348:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972106.002357:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585792, portal 10 00000400:00000010:3.0:1788972106.002366:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788972106.002369:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189585792, offset 0 00000400:00000200:3.0:1788972106.002380:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972106.002390:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972106.002400:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972106.002406:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972106.002411:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972106.002416:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972106.002419:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972106.002423:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [567:567:256:4294967295] 192.168.203.150@tcp LPNI seq info [567:567:8:4294967295] 00000400:00000200:3.0:1788972106.002428:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972106.002439:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972106.002447:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972106.002453:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67d00. 00000800:00000200:3.0:1788972106.002460:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972106.002468:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972106.002473:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972106.002503:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972106.002510:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09d20. 00000100:00000010:3.0:1788972106.002528:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09060. 02000000:00000010:3.0:1788972106.002551:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e400. 00000400:00000200:3.0:1788972106.002556:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972106.002558:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972106.002562:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972106.002565:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585664, portal 25 00000400:00000010:3.0:1788972106.002568:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873000. 00000100:00000200:3.0:1788972106.002571:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189585664, offset 0 00000400:00000200:3.0:1788972106.002576:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972106.002579:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972106.002589:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972106.002592:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972106.002595:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972106.002597:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972106.002599:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972106.002601:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [568:568:255:4294967295] 192.168.203.150@tcp LPNI seq info [568:568:7:4294967295] 00000400:00000200:3.0:1788972106.002605:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972106.002610:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972106.002625:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972106.002629:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67300. 00000800:00000200:3.0:1788972106.002633:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972106.002637:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972106.002639:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972106.002644:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972106.002657:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788972106.002660:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972106.002663:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972106.002667:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788972106.002670:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189585920, portal 10 00000400:00000010:3.0:1788972106.002675:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873550. 00000100:00000200:3.0:1788972106.002678:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189585920, offset 0 00000400:00000200:3.0:1788972106.002682:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972106.002685:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972106.002688:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972106.002690:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972106.002693:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972106.002695:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972106.002697:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972106.002700:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [569:569:254:4294967295] 192.168.203.150@tcp LPNI seq info [569:569:6:4294967295] 00000400:00000200:3.0:1788972106.002703:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972106.002708:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972106.002719:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972106.002722:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972106.002726:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972106.002730:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972106.002733:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972106.002738:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972106.003177:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972106.003184:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67d00. 00000400:00000200:2.0:1788972106.003193:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972106.003200:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972106.003207:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28 00000400:00000010:2.0:1788972106.003210:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28. 00000100:00000200:2.0:1788972106.003220:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189585792/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972106.003247:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972106.003249:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67300. 00000400:00000200:2.0:1788972106.003252:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972106.003262:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972106.003265:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873000 00000400:00000010:2.0:1788972106.003267:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873000. 00000100:00000200:2.0:1788972106.003272:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793100 x1875872189585664/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972106.003353:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972106.003357:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972106.003360:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972106.003363:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972106.003366:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873550 00000400:00000010:2.0:1788972106.003368:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873550. 00000100:00000200:2.0:1788972106.003373:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790000 x1875872189585920/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972106.009395:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.009409:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972106.009419:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.009425:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.009432:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.009436:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811980 00000400:00000200:2.0:1788972106.009443:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.009451:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1be5 [1] + 224 00000400:00000200:2.0:1788972106.009467:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972106.009472:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972106.009552:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189585792/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972106.009754:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972106.009760:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:3.0:1788972106.009763:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972106.009771:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189585792/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972106.009782:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189585792/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972106.009856:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f400. 02000000:00000010:3.0:1788972106.009864:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972106.009869:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 00000400:00000200:2.0:1788972106.011427:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011437:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972106.011446:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011452:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011457:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011536:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811a00 00000400:00000200:2.0:1788972106.011542:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011549:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c05 [1] + 224 00000400:00000200:2.0:1788972106.011563:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972106.011568:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972106.011582:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790000 x1875872189585920/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972106.011673:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011676:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972106.011680:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011683:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011686:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011688:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811900 00000400:00000200:2.0:1788972106.011692:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972106.011696:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1bf5 [1] + 224 00000400:00000200:2.0:1788972106.011706:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972106.011710:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972106.011717:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793100 x1875872189585664/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972106.016317:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972106.016337:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:3.0:1788972106.016340:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788972106.016360:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790000 x1875872189585920/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972106.016373:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790000 x1875872189585920/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972106.016417:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:3.0:1788972106.016426:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788972106.016431:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000. 00000400:00000010:3.0:1788972106.016441:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972106.016444:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972106.016446:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972106.016524:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793100 x1875872189585664/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972106.016534:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793100 x1875872189585664/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972122 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972106.016547:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e400. 02000000:00000010:3.0:1788972106.016550:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788972106.016553:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100. 00000100:00000010:2.0:1788972107.026021:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09d20. 00000100:00000010:3.0:1788972107.026063:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09060. 00000100:00000010:3.0:1788972111.122028:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100. 02000000:00000010:3.0:1788972111.122054:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788972111.122093:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000. 02000000:00000010:3.0:1788972111.122099:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788972111.122107:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793b80. 02000000:00000010:3.0:1788972111.122111:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 02000000:00000010:3.0:1788972111.122140:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f400. 00000400:00000200:3.0:1788972111.122150:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972111.122153:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a400. 00000400:00000010:3.0:1788972111.122155:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000200:2.0:1788972111.122163:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972111.122165:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000400:00000010:2.0:1788972111.122169:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00. 00000100:00000200:3.0:1788972111.122174:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586176, portal 10 00000400:00000010:2.0:1788972111.122176:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972111.122182:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586048, portal 25 00000400:00000010:3.0:1788972111.122186:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000400:00000010:2.0:1788972111.122188:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8. 00000100:00000200:3.0:1788972111.122192:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189586176, offset 0 00000100:00000200:2.0:1788972111.122194:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189586048, offset 0 00000400:00000200:3.0:1788972111.122321:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972111.122332:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.122339:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972111.122344:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972111.122348:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972111.122352:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972111.122355:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972111.122358:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [570:570:256:4294967295] 192.168.203.150@tcp LPNI seq info [570:570:8:4294967295] 00000400:00000200:3.0:1788972111.122362:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.122370:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972111.122402:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972111.122409:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972111.122416:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972111.122423:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972111.122427:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972111.122443:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972111.122448:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09060. 00000100:00000010:3.0:1788972111.122458:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09a20. 00000400:00000200:2.0:1788972111.122576:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972111.122581:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.122586:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972111.122596:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972111.122598:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972111.122601:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972111.122603:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972111.122606:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [571:571:255:4294967295] 192.168.203.150@tcp LPNI seq info [571:571:7:4294967295] 00000400:00000200:2.0:1788972111.122610:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.122617:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972111.122622:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972111.122626:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972111.122630:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972111.122635:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972111.122638:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972111.122643:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972111.122656:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a600. 00000400:00000200:2.0:1788972111.122660:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972111.122663:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972111.122750:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972111.122754:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586304, portal 10 00000400:00000010:2.0:1788972111.122759:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18. 00000100:00000200:2.0:1788972111.122761:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189586304, offset 0 00000400:00000200:2.0:1788972111.122766:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972111.122770:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.122773:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972111.122776:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972111.122779:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972111.122781:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972111.122783:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972111.122785:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [572:572:254:4294967295] 192.168.203.150@tcp LPNI seq info [572:572:6:4294967295] 00000400:00000200:2.0:1788972111.122789:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.122795:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972111.122809:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972111.122830:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63200. 00000800:00000200:2.0:1788972111.122835:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972111.122839:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972111.122842:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972111.122847:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972111.123066:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972111.123071:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:3.0:1788972111.123077:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972111.123083:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972111.123088:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:3.0:1788972111.123090:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788972111.123099:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790000 x1875872189586176/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972111.123123:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972111.123126:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:3.0:1788972111.123129:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972111.123132:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972111.123135:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8 00000400:00000010:3.0:1788972111.123136:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8. 00000100:00000200:3.0:1788972111.123141:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793100 x1875872189586048/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972111.123211:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972111.123215:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63200. 00000400:00000200:3.0:1788972111.123224:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972111.123228:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972111.123231:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18 00000400:00000010:3.0:1788972111.123232:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18. 00000100:00000200:3.0:1788972111.123237:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793b80 x1875872189586304/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972111.126845:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126856:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972111.126866:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126879:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126884:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126888:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811b00 00000400:00000200:2.0:1788972111.126894:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126900:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c15 [1] + 224 00000400:00000200:2.0:1788972111.126912:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972111.126916:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972111.126929:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790000 x1875872189586176/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972111.126975:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126978:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972111.126983:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126985:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126988:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126991:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811b80 00000400:00000200:2.0:1788972111.126994:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972111.126998:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c35 [1] + 224 00000400:00000200:2.0:1788972111.127009:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972111.127012:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972111.127018:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793b80 x1875872189586304/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972111.127072:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:2.0:1788972111.127076:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0 00000400:00000010:2.0:1788972111.127079:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972111.127085:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793b80 x1875872189586304/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972111.127094:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793b80 x1875872189586304/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972111.127125:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a600. 02000000:00000010:2.0:1788972111.127134:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788972111.127138:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793b80. 00000400:00000010:3.0:1788972111.127221:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972111.127226:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972111.127228:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972111.127234:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790000 x1875872189586176/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972111.127243:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790000 x1875872189586176/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972111.127258:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f400. 02000000:00000010:3.0:1788972111.127262:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67600. 00000100:00000010:3.0:1788972111.127265:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000. 00000400:00000200:3.0:1788972111.130592:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.130602:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972111.130610:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.130615:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.130620:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.130624:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811a80 00000400:00000200:3.0:1788972111.130629:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972111.130635:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c1d [1] + 224 00000400:00000200:3.0:1788972111.130647:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972111.130652:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972111.130674:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793100 x1875872189586048/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972111.132908:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00. 00000400:00000200:2.0:1788972111.132918:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68 00000400:00000010:2.0:1788972111.132921:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972111.132933:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793100 x1875872189586048/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972111.132945:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793100 x1875872189586048/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972127 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972111.132972:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a400. 02000000:00000010:2.0:1788972111.132979:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788972111.132983:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100. 00000100:00000010:2.0:1788972112.145970:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09060. 00000100:00000010:3.0:1788972112.146016:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09a20. 00000001:02000400:2.0:1788972113.373952:0:115060:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_303: @@@@@@ FAIL: Failed to cleanup before test execution Debug log: 4367 lines, 4367 kept, 0 dropped, 0 bad.