00000100:00000010:0.0F:1788972495.123017:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b100. 02000000:00000010:0.0:1788972495.123047:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7b00. 00000100:00000010:0.0:1788972495.123082:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788972495.123087:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7800. 00000100:00000010:0.0:1788972495.123093:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972495.123097:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7700. 02000000:00000010:0.0:1788972495.123125:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cee000. 00000400:00000200:0.0:1788972495.123134:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972495.123138:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4aa8. 00000400:00000010:0.0:1788972495.123153:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8198. 00000100:00000200:0.0:1788972495.123160:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615232, portal 10 00000400:00000010:0.0:1788972495.123167:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8ee0. 00000100:00000200:0.0:1788972495.123170:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189615232, offset 0 00000400:00000200:0.0:1788972495.123179:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972495.123188:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972495.123195:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972495.123201:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972495.123204:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972495.123208:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972495.123211:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972495.123214:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [797:797:256:4294967295] 192.168.203.150@tcp LPNI seq info [797:797:8:4294967295] 00000400:00000200:0.0:1788972495.123218:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972495.123227:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972495.123234:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972495.123247:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7100. 00000800:00000200:0.0:1788972495.123255:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972495.123262:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972495.123267:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972495.123277:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972495.123282:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391720. 00000100:00000010:0.0:1788972495.123291:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d1391d80. 02000000:00000010:0.0:1788972495.123309:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cefa00. 00000400:00000200:0.0:1788972495.123318:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972495.123320:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972495.123324:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8660. 00000100:00000200:0.0:1788972495.123326:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615104, portal 25 00000400:00000010:0.0:1788972495.123329:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58. 00000100:00000200:0.0:1788972495.123331:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189615104, offset 0 00000400:00000200:0.0:1788972495.123336:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972495.123339:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972495.123343:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972495.123345:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972495.123347:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972495.123350:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972495.123352:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972495.123354:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [798:798:255:4294967295] 192.168.203.150@tcp LPNI seq info [798:798:7:4294967295] 00000400:00000200:0.0:1788972495.123358:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0F:1788972495.123360:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1788972495.123363:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000010:3.0:1788972495.123368:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7100. 00000800:00000200:0.0:1788972495.123368:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972495.123371:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7c00. 00000400:00000200:3.0:1788972495.123372:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1788972495.123374:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:3.0:1788972495.123376:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000800:00000200:0.0:1788972495.123378:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972495.123381:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972495.123385:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972495.123397:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7ceea00. 00000400:00000200:0.0:1788972495.123400:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972495.123403:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972495.123405:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788972495.123408:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615360, portal 10 00000400:00000010:0.0:1788972495.123412:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8dd0. 00000100:00000200:0.0:1788972495.123414:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189615360, offset 0 00000400:00000200:0.0:1788972495.123418:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972495.123421:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972495.123430:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972495.123432:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972495.123435:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972495.123437:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972495.123439:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972495.123442:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [799:799:254:4294967295] 192.168.203.150@tcp LPNI seq info [799:799:6:4294967295] 00000400:00000200:0.0:1788972495.123446:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972495.123452:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000400:00000200:3.0:1788972495.123456:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8ee0 00000800:00000200:0.0:1788972495.123456:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000010:3.0:1788972495.123458:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8ee0. 00000100:00000200:3.0:1788972495.123468:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88000 x1875872189615232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1788972495.123473:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7d00. 00000800:00000200:0.0:1788972495.123477:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972495.123480:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972495.123483:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972495.123488:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972495.123493:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972495.123496:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7c00. 00000400:00000200:3.0:1788972495.123498:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972495.123501:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972495.123504:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58 00000400:00000010:3.0:1788972495.123506:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58. 00000100:00000200:3.0:1788972495.123510:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b100 x1875872189615104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972511 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972495.123538:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972495.123541:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7d00. 00000400:00000200:3.0:1788972495.123543:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972495.123546:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972495.123549:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8dd0 00000400:00000010:3.0:1788972495.123550:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8dd0. 00000100:00000200:3.0:1788972495.123554:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189615360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0F:1788972495.138324:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138348:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972495.138358:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138364:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138369:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138373:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887818c00 00000400:00000200:2.0:1788972495.138378:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138535:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x273d [1] + 224 00000400:00000200:2.0:1788972495.138548:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972495.138552:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972495.138565:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b100 x1875872189615104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972495.138592:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138595:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972495.138600:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138602:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138605:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138608:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818c80 00000400:00000200:2.0:1788972495.138612:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972495.138615:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2735 [1] + 224 00000400:00000200:2.0:1788972495.138634:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972495.138638:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972495.138646:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88000 x1875872189615232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0F:1788972495.139089:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:1.0:1788972495.139102:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8660 00000400:00000010:1.0:1788972495.139105:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8660. 00000100:00000200:1.0:1788972495.139111:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b100 x1875872189615104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972495.139120:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b100 x1875872189615104/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972495.139152:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4aa8. 00000400:00000200:1.0:1788972495.139156:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8198 00000400:00000010:1.0:1788972495.139158:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8198. 00000100:00000200:1.0:1788972495.139162:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88000 x1875872189615232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972495.139169:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88000 x1875872189615232/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972495.139189:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cefa00. 02000000:00000010:1.0:1788972495.139196:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7b00. 00000100:00000010:1.0:1788972495.139199:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b100. 02000000:00000010:1.0:1788972495.139203:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cee000. 02000000:00000010:1.0:1788972495.139205:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7800. 00000100:00000010:1.0:1788972495.139207:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88000. 00000400:00000200:3.0:1788972495.146515:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972495.146524:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972495.146533:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972495.146537:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972495.146542:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972495.146546:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818d00 00000400:00000200:3.0:1788972495.146551:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972495.146558:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2745 [1] + 224 00000400:00000200:3.0:1788972495.146581:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972495.146586:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972495.146601:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189615360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972495.146802:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972495.146808:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:0.0:1788972495.146811:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788972495.146838:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189615360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972495.146858:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189615360/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972511 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972495.146889:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7ceea00. 02000000:00000010:0.0:1788972495.146896:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7700. 00000100:00000010:0.0:1788972495.146900:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 00000001:02000400:3.0:1788972495.169945:0:125913:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 550: DD-off: keep route up when gw reachable via another local NI ========================================================== 12:48:13 (1788972493) 00000100:00000010:2.0:1788972496.145899:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391720. 00000100:00000010:3.0:1788972496.151521:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d1391d80. 00000001:02000400:0.0:1788972497.083163:0:126051:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_550 Need at least 2 local interfaces for multi-rail 00000001:02000400:3.0:1788972498.900850:0:126205:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 600: lnet_legacy2yaml: basic ip2nets conversion ========================================================== 12:48:17 (1788972497) 00000100:00000010:0.0:1788972500.242105:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972500.242142:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7200. 00000100:00000010:0.0:1788972500.242172:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972500.242178:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972500.242199:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972500.242205:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7700. 02000000:00000010:0.0:1788972500.242237:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4583200. 00000400:00000200:0.0:1788972500.242247:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972500.242251:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972500.242259:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788972500.242266:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615488, portal 25 00000400:00000010:0.0:1788972500.242272:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8aa0. 00000100:00000200:0.0:1788972500.242276:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189615488, offset 0 00000400:00000200:0.0:1788972500.242286:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972500.242295:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972500.242303:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972500.242308:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972500.242312:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972500.242315:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972500.242318:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972500.242322:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [800:800:256:4294967295] 192.168.203.150@tcp LPNI seq info [800:800:8:4294967295] 00000400:00000200:0.0:1788972500.242326:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972500.242336:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972500.242342:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972500.242347:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7600. 00000800:00000200:0.0:1788972500.242353:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972500.242361:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972500.242365:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972500.242385:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972500.242390:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73e0183720. 00000100:00000010:0.0:1788972500.242400:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73e0183d20. 02000000:00000010:0.0:1788972500.242411:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c137d200. 00000400:00000200:0.0:1788972500.242414:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972500.242417:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4210. 00000400:00000010:0.0:1788972500.242420:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788972500.242429:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615744, portal 10 00000400:00000010:0.0:1788972500.242435:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972500.242437:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189615744, offset 0 00000400:00000200:0.0:1788972500.242442:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972500.242446:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972500.242450:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972500.242453:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972500.242455:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972500.242457:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972500.242460:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972500.242462:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [801:801:255:4294967295] 192.168.203.150@tcp LPNI seq info [801:801:7:4294967295] 00000400:00000200:0.0:1788972500.242466:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972500.242472:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972500.242476:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972500.242479:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7800. 00000800:00000200:0.0:1788972500.242483:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972500.242487:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972500.242490:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972500.242496:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972500.242514:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d935ce00. 00000400:00000200:0.0:1788972500.242517:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972500.242520:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4d68. 00000400:00000010:0.0:1788972500.242523:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972500.242525:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615616, portal 10 00000400:00000010:0.0:1788972500.242530:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8330. 00000100:00000200:0.0:1788972500.242532:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189615616, offset 0 00000400:00000200:0.0:1788972500.242537:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972500.242540:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972500.242543:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972500.242546:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972500.242548:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972500.242556:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972500.242558:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972500.242561:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [802:802:254:4294967295] 192.168.203.150@tcp LPNI seq info [802:802:6:4294967295] 00000400:00000200:0.0:1788972500.242564:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972500.242571:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972500.242575:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972500.242578:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7b00. 00000800:00000200:0.0:1788972500.242582:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972500.242585:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972500.242588:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972500.242593:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972500.242687:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972500.242692:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7600. 00000400:00000200:2.0:1788972500.242697:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972500.242702:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972500.242708:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8aa0 00000400:00000010:2.0:1788972500.242710:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8aa0. 00000100:00000200:2.0:1788972500.242725:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189615488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972516 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972500.242748:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972500.242750:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7800. 00000400:00000200:2.0:1788972500.242753:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972500.242755:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972500.242758:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088 00000400:00000010:2.0:1788972500.242759:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088. 00000100:00000200:2.0:1788972500.242764:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189615744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972500.242856:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972500.242859:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7b00. 00000400:00000200:2.0:1788972500.242862:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972500.242866:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972500.242869:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8330 00000400:00000010:2.0:1788972500.242871:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8330. 00000100:00000200:2.0:1788972500.242876:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189615616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972500.250203:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250254:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972500.250264:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250269:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250274:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250279:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887818d80 00000400:00000200:3.0:1788972500.250284:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250291:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x274d [1] + 224 00000400:00000200:3.0:1788972500.250303:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972500.250308:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972500.250322:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189615488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972500.250361:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250365:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972500.250369:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250372:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250374:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250377:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818e80 00000400:00000200:3.0:1788972500.250380:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250384:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2755 [1] + 224 00000400:00000200:3.0:1788972500.250407:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972500.250410:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972500.250417:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189615744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972500.250433:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250436:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972500.250440:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250442:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250445:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250447:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818e00 00000400:00000200:3.0:1788972500.250451:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972500.250454:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x275d [1] + 224 00000400:00000200:3.0:1788972500.250461:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972500.250464:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972500.250470:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189615616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972500.250572:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4d68. 00000400:00000200:0.0:1788972500.250578:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28 00000400:00000010:0.0:1788972500.250580:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972500.250586:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189615616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972500.250595:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189615616/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972500.250624:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972500.250627:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:0.0:1788972500.250629:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788972500.250633:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189615488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972500.250641:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189615488/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972500.250650:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4210. 00000400:00000200:0.0:1788972500.250652:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38 00000400:00000010:0.0:1788972500.250653:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788972500.250656:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189615744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972500.250675:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189615744/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972516 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972500.250692:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d935ce00. 02000000:00000010:0.0:1788972500.250699:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972500.250703:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972500.250707:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4583200. 02000000:00000010:0.0:1788972500.250711:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7200. 00000100:00000010:0.0:1788972500.250713:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972500.250716:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c137d200. 02000000:00000010:0.0:1788972500.250719:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7700. 00000100:00000010:0.0:1788972500.250721:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 00000100:00000010:3.0:1788972501.265977:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73e0183d20. 00000100:00000010:2.0:1788972501.276394:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73e0183720. 00000100:00000010:2.0:1788972505.394900:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972505.394921:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387d00. 02000000:00000010:2.0:1788972505.394981:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cefc00. 00000400:00000200:2.0:1788972505.394991:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972505.394996:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:2.0:1788972505.395003:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972505.395010:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189615872, portal 25 00000400:00000010:2.0:1788972505.395015:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972505.395018:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189615872, offset 0 00000400:00000200:2.0:1788972505.395026:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972505.395035:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972505.395041:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972505.395060:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972505.395064:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972505.395068:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972505.395080:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972505.395084:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [803:803:256:4294967295] 192.168.203.150@tcp LPNI seq info [803:803:8:4294967295] 00000400:00000200:2.0:1788972505.395089:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972505.395099:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972505.395105:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972505.395110:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387100. 00000800:00000200:2.0:1788972505.395117:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972505.395125:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972505.395130:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972505.395144:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972505.395149:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651eae0. 00000100:00000010:2.0:1788972505.395159:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e5a0. 00000800:00000200:2.0:1788972505.395253:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972505.395257:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387100. 00000400:00000200:2.0:1788972505.395261:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972505.395265:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972505.395271:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:2.0:1788972505.395274:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972505.395283:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189615872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1788972505.395310:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972505.395316:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387100. 02000000:00000010:2.0:1788972505.395337:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef600. 00000400:00000200:2.0:1788972505.395341:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972505.395343:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126528. 00000400:00000010:2.0:1788972505.395346:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972505.395349:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616000, portal 10 00000400:00000010:2.0:1788972505.395354:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972505.395365:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189616000, offset 0 00000400:00000200:2.0:1788972505.395370:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972505.395374:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972505.395377:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972505.395380:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972505.395382:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972505.395385:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972505.395386:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972505.395389:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [804:804:256:4294967295] 192.168.203.150@tcp LPNI seq info [804:804:8:4294967295] 00000400:00000200:2.0:1788972505.395392:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972505.395397:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972505.395401:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972505.395403:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387b00. 00000800:00000200:2.0:1788972505.395407:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972505.395410:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972505.395413:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972505.395422:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972505.395436:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972505.395441:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387200. 02000000:00000010:2.0:1788972505.395462:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cee400. 00000400:00000200:2.0:1788972505.395466:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972505.395468:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972505.395471:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972505.395474:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616128, portal 10 00000400:00000010:2.0:1788972505.395478:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788972505.395480:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189616128, offset 0 00000400:00000200:2.0:1788972505.395484:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972505.395488:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972505.395492:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972505.395494:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972505.395496:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972505.395503:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972505.395505:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972505.395508:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [805:805:255:4294967295] 192.168.203.150@tcp LPNI seq info [805:805:7:4294967295] 00000400:00000200:2.0:1788972505.395512:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972505.395517:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972505.395522:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972505.395525:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387900. 00000800:00000200:2.0:1788972505.395528:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972505.395532:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972505.395534:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972505.395539:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.404489:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.404500:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972505.404509:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.404513:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.404519:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.404522:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887818f00 00000400:00000200:3.0:1788972505.404528:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.404535:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2765 [1] + 224 00000400:00000200:3.0:1788972505.404548:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972505.404553:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972505.404567:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189615872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972505.404633:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972505.404637:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387b00. 00000400:00000200:3.0:1788972505.404641:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972505.404646:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972505.404649:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:3.0:1788972505.404652:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:3.0:1788972505.404658:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189616000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972505.404694:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972505.404697:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387900. 00000400:00000200:3.0:1788972505.404700:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972505.404704:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972505.404706:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:3.0:1788972505.404708:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:3.0:1788972505.404714:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189616128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972505.404752:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:3.0:1788972505.404755:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:3.0:1788972505.404758:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:3.0:1788972505.404762:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189615872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972505.404771:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189615872/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972505.404795:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cefc00. 02000000:00000010:3.0:1788972505.404800:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387d00. 00000100:00000010:3.0:1788972505.404804:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 00000400:00000200:3.0:1788972505.409181:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.409191:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972505.409200:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.409221:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.409226:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.409230:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818f80 00000400:00000200:3.0:1788972505.409235:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.409241:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2775 [1] + 224 00000400:00000200:3.0:1788972505.409254:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972505.409258:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972505.409272:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189616000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972505.409311:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126528. 00000400:00000200:3.0:1788972505.409316:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:3.0:1788972505.409318:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:3.0:1788972505.409324:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189616000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972505.409331:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189616000/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972505.409362:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef600. 02000000:00000010:3.0:1788972505.409368:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387100. 00000100:00000010:3.0:1788972505.409372:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 00000400:00000200:3.0:1788972505.413697:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.413706:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972505.413715:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.413720:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.413725:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.413729:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819000 00000400:00000200:3.0:1788972505.413734:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972505.413740:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2785 [1] + 224 00000400:00000200:3.0:1788972505.413792:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972505.413797:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972505.413810:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189616128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972505.414102:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:3.0:1788972505.414107:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:3.0:1788972505.414110:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:3.0:1788972505.414116:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189616128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972505.414125:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189616128/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972521 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972505.414157:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cee400. 02000000:00000010:3.0:1788972505.414162:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387200. 00000100:00000010:3.0:1788972505.414167:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 00000100:00000010:3.0:1788972506.387979:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e5a0. 00000100:00000010:2.0:1788972506.388811:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651eae0. 00000001:02000400:1.0:1788972506.792805:0:126786:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 601: lnet_legacy2yaml: basic routes conversion ========================================================== 12:48:25 (1788972505) 00000100:00000010:3.0:1788972510.482016:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788972510.482049:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bb00. 02000000:00000010:2.0:1788972510.482116:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5000. 00000400:00000200:2.0:1788972510.482126:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972510.482130:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c08. 00000400:00000010:2.0:1788972510.482136:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972510.482142:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616256, portal 25 00000400:00000010:2.0:1788972510.482149:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972510.482152:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189616256, offset 0 00000400:00000200:2.0:1788972510.482160:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972510.482169:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.482177:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972510.482182:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972510.482186:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972510.482190:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972510.482193:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972510.482197:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [806:806:256:4294967295] 192.168.203.150@tcp LPNI seq info [806:806:8:4294967295] 00000400:00000200:2.0:1788972510.482201:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.482210:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972510.482216:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972510.482234:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387c00. 00000800:00000200:2.0:1788972510.482241:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972510.482250:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972510.482254:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972510.482271:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972510.482276:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78060. 00000100:00000010:2.0:1788972510.482287:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78b40. 00000100:00000010:3.0:1788972510.482332:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788972510.482340:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 02000000:00000010:2.0:1788972510.485478:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4000. 00000400:00000200:2.0:1788972510.485495:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972510.485501:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972510.485508:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972510.485514:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616384, portal 10 00000400:00000010:2.0:1788972510.485521:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972510.485524:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189616384, offset 0 00000400:00000200:2.0:1788972510.485533:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972510.485541:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.485549:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972510.485554:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972510.485557:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972510.485561:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972510.485563:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972510.485567:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [807:807:255:4294967295] 192.168.203.150@tcp LPNI seq info [807:807:7:4294967295] 00000400:00000200:2.0:1788972510.485571:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.485580:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972510.485585:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972510.485659:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387300. 00000800:00000200:2.0:1788972510.485666:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972510.485674:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972510.485679:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972510.485687:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972510.485753:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972510.485757:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387c00. 00000400:00000200:2.0:1788972510.485764:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972510.485769:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972510.485774:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788972510.485776:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972510.485786:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790380 x1875872189616256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972510.486562:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972510.486566:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387300. 00000400:00000200:2.0:1788972510.486570:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972510.486575:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972510.486579:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:2.0:1788972510.486581:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972510.486586:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189616384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1788972510.490672:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788972510.490693:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b900. 02000000:00000010:3.0:1788972510.492544:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003400. 00000400:00000200:3.0:1788972510.492558:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972510.492565:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436cb8. 00000400:00000010:3.0:1788972510.492572:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972510.492579:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616512, portal 10 00000400:00000010:3.0:1788972510.492585:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873aa0. 00000100:00000200:3.0:1788972510.492588:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189616512, offset 0 00000400:00000200:3.0:1788972510.492596:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972510.492603:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972510.492611:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972510.492615:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972510.492619:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972510.492635:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972510.492638:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972510.492642:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [808:808:256:4294967295] 192.168.203.150@tcp LPNI seq info [808:808:8:4294967295] 00000400:00000200:3.0:1788972510.492647:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972510.492656:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972510.492662:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972510.492666:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:3.0:1788972510.492673:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972510.492680:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972510.492686:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972510.494182:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.497440:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.497450:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972510.497459:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.497464:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.497470:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.497474:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819080 00000400:00000200:2.0:1788972510.497479:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.497485:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2795 [1] + 224 00000400:00000200:2.0:1788972510.497497:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972510.497501:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972510.497515:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790380 x1875872189616256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972510.498080:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972510.498085:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:2.0:1788972510.498091:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972510.498109:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972510.498114:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873aa0 00000400:00000010:2.0:1788972510.498116:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873aa0. 00000100:00000200:2.0:1788972510.498124:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792680 x1875872189616512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972510.498148:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.498152:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972510.498156:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.498159:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.498162:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.498165:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819100 00000400:00000200:2.0:1788972510.498169:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.498173:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x27a5 [1] + 224 00000400:00000200:2.0:1788972510.498184:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972510.498187:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972510.498193:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189616384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972510.500719:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c08. 00000400:00000200:3.0:1788972510.500728:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:3.0:1788972510.500730:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:3.0:1788972510.500742:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790380 x1875872189616256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972510.500753:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790380 x1875872189616256/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972510.500779:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5000. 02000000:00000010:3.0:1788972510.500785:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bb00. 00000100:00000010:3.0:1788972510.500788:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380. 00000400:00000010:3.0:1788972510.500797:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:3.0:1788972510.500799:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:3.0:1788972510.500801:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:3.0:1788972510.500806:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189616384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972510.500831:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189616384/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972510.500847:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4000. 02000000:00000010:3.0:1788972510.500864:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:3.0:1788972510.500867:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 00000400:00000200:2.0:1788972510.503065:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.503074:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972510.503084:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.503088:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.503094:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.503097:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819180 00000400:00000200:2.0:1788972510.503102:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972510.503108:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x27b5 [1] + 224 00000400:00000200:2.0:1788972510.503126:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972510.503131:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972510.503146:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792680 x1875872189616512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972510.506185:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436cb8. 00000400:00000200:3.0:1788972510.506198:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972510.506200:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972510.506213:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792680 x1875872189616512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972510.506225:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792680 x1875872189616512/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972526 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972510.506259:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003400. 02000000:00000010:3.0:1788972510.506266:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b900. 00000100:00000010:3.0:1788972510.506270:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792680. 00000100:00000010:2.0:1788972511.505913:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78060. 00000100:00000010:3.0:1788972511.506412:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78b40. 00000001:02000400:1.0:1788972514.710207:0:127366:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 602: lnet_legacy2yaml: routes with priority ========================================================== 12:48:33 (1788972513) 00000100:00000010:3.0:1788972515.603232:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788972515.603255:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b300. 00000100:00000010:3.0:1788972515.603286:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788972515.603291:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b700. 00000100:00000010:3.0:1788972515.603297:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788972515.603300:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 02000000:00000010:3.0:1788972515.603331:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625000. 00000400:00000200:3.0:1788972515.603341:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972515.603346:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436cb8. 00000400:00000010:3.0:1788972515.603353:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972515.603360:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616640, portal 25 00000400:00000010:3.0:1788972515.603364:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8. 00000100:00000200:3.0:1788972515.603367:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189616640, offset 0 00000400:00000200:3.0:1788972515.603376:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972515.603384:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.603392:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972515.603397:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972515.603401:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972515.603418:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972515.603421:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972515.603425:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [809:809:256:4294967295] 192.168.203.150@tcp LPNI seq info [809:809:8:4294967295] 00000400:00000200:3.0:1788972515.603429:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.603438:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972515.603444:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972515.603450:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:3.0:1788972515.603456:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972515.603463:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972515.603468:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972515.603483:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972515.603488:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d180. 00000100:00000010:3.0:1788972515.603499:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d660. 02000000:00000010:3.0:1788972515.603510:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625400. 00000400:00000200:3.0:1788972515.603514:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972515.603517:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436840. 00000400:00000010:3.0:1788972515.603520:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972515.603523:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616896, portal 10 00000400:00000010:3.0:1788972515.603529:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788972515.603531:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189616896, offset 0 00000400:00000200:3.0:1788972515.603535:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972515.603539:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.603542:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972515.603545:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972515.603547:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972515.603549:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972515.603552:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972515.603554:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [810:810:255:4294967295] 192.168.203.150@tcp LPNI seq info [810:810:7:4294967295] 00000400:00000200:3.0:1788972515.603558:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.603564:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972515.603577:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972515.603581:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:3.0:1788972515.603586:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972515.603590:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972515.603592:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972515.603598:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972515.603616:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625600. 00000400:00000200:3.0:1788972515.603619:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972515.603622:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972515.603625:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972515.603629:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189616768, portal 10 00000400:00000010:3.0:1788972515.603633:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788972515.603635:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189616768, offset 0 00000400:00000200:3.0:1788972515.603639:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972515.603642:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.603646:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972515.603648:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972515.603650:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972515.603652:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972515.603654:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972515.603657:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [811:811:254:4294967295] 192.168.203.150@tcp LPNI seq info [811:811:6:4294967295] 00000400:00000200:3.0:1788972515.603661:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.603666:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972515.603671:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972515.603674:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1be00. 00000800:00000200:3.0:1788972515.603678:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972515.603682:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972515.603685:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1be00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972515.603689:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972515.603752:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972515.603762:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:3.0:1788972515.603768:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972515.603773:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972515.603779:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8 00000400:00000010:3.0:1788972515.603782:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8. 00000100:00000200:3.0:1788972515.603792:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792680 x1875872189616640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972515.603848:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972515.603851:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:3.0:1788972515.603854:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972515.603857:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972515.603860:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770 00000400:00000010:3.0:1788972515.603862:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788972515.603867:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790380 x1875872189616896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972515.603932:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972515.603935:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1be00. 00000400:00000200:3.0:1788972515.603938:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972515.603941:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972515.603944:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28 00000400:00000010:3.0:1788972515.603945:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788972515.603950:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189616768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972515.608440:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608449:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972515.608458:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608463:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608468:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608472:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819200 00000400:00000200:3.0:1788972515.608477:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608482:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x27c5 [1] + 224 00000400:00000200:3.0:1788972515.608521:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972515.608527:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972515.608541:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792680 x1875872189616640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972515.608708:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608712:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972515.608716:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608718:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608782:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608785:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819300 00000400:00000200:3.0:1788972515.608789:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608793:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x27d5 [1] + 224 00000400:00000200:3.0:1788972515.608799:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972515.608803:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972515.608809:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790380 x1875872189616896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972515.608839:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608843:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972515.608847:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608849:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608852:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608854:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819280 00000400:00000200:3.0:1788972515.608857:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972515.608860:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x27e5 [1] + 224 00000400:00000200:3.0:1788972515.608868:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972515.608871:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972515.608876:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189616768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972515.608905:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972515.608916:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:3.0:1788972515.608918:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972515.608924:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189616768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972515.608932:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189616768/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972515.608958:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436cb8. 00000400:00000200:3.0:1788972515.608961:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972515.608963:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972515.608967:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792680 x1875872189616640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972515.608976:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792680 x1875872189616640/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972515.608986:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436840. 00000400:00000200:3.0:1788972515.608988:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972515.608989:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972515.608993:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790380 x1875872189616896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972515.609626:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790380 x1875872189616896/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972531 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972515.609649:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625600. 02000000:00000010:3.0:1788972515.609656:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b700. 00000100:00000010:3.0:1788972515.609660:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788972515.609663:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625000. 02000000:00000010:3.0:1788972515.609666:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b300. 00000100:00000010:3.0:1788972515.609667:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788972515.609670:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625400. 02000000:00000010:3.0:1788972515.609672:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:3.0:1788972515.609674:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380. 00000100:00000010:3.0:1788972516.626308:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d660. 00000100:00000010:2.0:1788972516.627101:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d180. 00000100:00000010:2.0:1788972520.724624:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972520.724643:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387e00. 00000100:00000010:2.0:1788972520.724673:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840df80. 02000000:00000010:2.0:1788972520.724678:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387700. 00000100:00000010:2.0:1788972520.724756:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972520.724763:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387400. 02000000:00000010:2.0:1788972520.724798:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3f3f200. 00000400:00000200:2.0:1788972520.724808:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972520.724812:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126420. 00000400:00000010:2.0:1788972520.724838:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972520.724845:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617024, portal 25 00000400:00000010:2.0:1788972520.724850:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972520.724853:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189617024, offset 0 00000400:00000200:2.0:1788972520.724862:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972520.724871:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.724879:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972520.724884:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972520.724889:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972520.724892:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972520.725001:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972520.725007:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [812:812:256:4294967295] 192.168.203.150@tcp LPNI seq info [812:812:8:4294967295] 00000400:00000200:2.0:1788972520.725013:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.725022:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972520.725028:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972520.725033:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387800. 00000800:00000200:2.0:1788972520.725039:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972520.725047:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972520.725059:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972520.725076:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972520.725081:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78000. 00000100:00000010:2.0:1788972520.725090:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78060. 02000000:00000010:2.0:1788972520.725102:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3f3e200. 00000400:00000200:2.0:1788972520.725106:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972520.725108:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126478. 00000400:00000010:2.0:1788972520.725111:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788972520.725114:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617280, portal 10 00000400:00000010:2.0:1788972520.725119:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972520.725121:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189617280, offset 0 00000400:00000200:2.0:1788972520.725125:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972520.725129:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.725132:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972520.725135:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972520.725137:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972520.725138:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972520.725141:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972520.725143:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [813:813:255:4294967295] 192.168.203.150@tcp LPNI seq info [813:813:7:4294967295] 00000400:00000200:2.0:1788972520.725146:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.725151:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972520.725154:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972520.725157:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387500. 00000800:00000200:2.0:1788972520.725161:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972520.725165:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972520.725167:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972520.725172:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972520.725188:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5ab2a00. 00000400:00000200:2.0:1788972520.725191:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972520.725194:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00. 00000400:00000010:2.0:1788972520.725197:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8. 00000100:00000200:2.0:1788972520.725206:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617152, portal 10 00000400:00000010:2.0:1788972520.725210:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972520.725213:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189617152, offset 0 00000400:00000200:2.0:1788972520.725217:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972520.725222:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.725225:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972520.725227:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972520.725230:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972520.725232:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972520.725234:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972520.725236:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [814:814:254:4294967295] 192.168.203.150@tcp LPNI seq info [814:814:6:4294967295] 00000400:00000200:2.0:1788972520.725239:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.725245:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972520.725249:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972520.725252:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387200. 00000800:00000200:2.0:1788972520.725255:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972520.725259:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972520.725261:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972520.725265:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972520.725309:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972520.725313:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387800. 00000400:00000200:2.0:1788972520.725319:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972520.725323:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972520.725328:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788972520.725330:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972520.725339:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189617024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972520.725358:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972520.725361:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387500. 00000400:00000200:2.0:1788972520.725363:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972520.725366:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972520.725375:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:2.0:1788972520.725377:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972520.725382:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189617280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972520.725444:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972520.725447:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387200. 00000400:00000200:2.0:1788972520.725450:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972520.725453:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972520.725456:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972520.725458:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972520.725462:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840df80 x1875872189617152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972520.739337:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.739347:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972520.739357:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.739361:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.739367:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.739371:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819380 00000400:00000200:2.0:1788972520.739377:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.739381:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x27f5 [1] + 224 00000400:00000200:2.0:1788972520.739398:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972520.739402:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972520.739415:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189617024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972520.740191:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126420. 00000400:00000200:2.0:1788972520.740199:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:2.0:1788972520.740202:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972520.740211:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189617024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972520.740221:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189617024/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972520.740257:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3f3f200. 02000000:00000010:2.0:1788972520.740266:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387e00. 00000100:00000010:2.0:1788972520.740271:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 00000400:00000200:2.0:1788972520.746109:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746119:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972520.746129:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746134:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746139:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746143:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819400 00000400:00000200:2.0:1788972520.746148:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746153:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2815 [1] + 224 00000400:00000200:2.0:1788972520.746164:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972520.746169:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972520.746183:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840df80 x1875872189617152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972520.746222:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746226:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972520.746230:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746232:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746234:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746237:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819480 00000400:00000200:2.0:1788972520.746240:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972520.746244:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2805 [1] + 224 00000400:00000200:2.0:1788972520.746254:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972520.746258:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972520.746264:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189617280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972520.746374:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00. 00000400:00000200:3.0:1788972520.746380:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8 00000400:00000010:3.0:1788972520.746382:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8. 00000100:00000200:3.0:1788972520.746389:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840df80 x1875872189617152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972520.746397:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840df80 x1875872189617152/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972520.746425:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126478. 00000400:00000200:3.0:1788972520.746427:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:3.0:1788972520.746429:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:3.0:1788972520.746433:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189617280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972520.746440:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189617280/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972536 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972520.746456:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5ab2a00. 02000000:00000010:3.0:1788972520.746463:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387700. 00000100:00000010:3.0:1788972520.746468:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840df80. 02000000:00000010:3.0:1788972520.746472:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3f3e200. 02000000:00000010:3.0:1788972520.746475:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387400. 00000100:00000010:3.0:1788972520.746477:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 00000100:00000010:3.0:1788972521.745889:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78060. 00000100:00000010:2.0:1788972521.746379:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78000. 00000001:02000400:2.0:1788972522.480167:0:127946:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 603: lnet_legacy2yaml: default hop flag ========================================================== 12:48:41 (1788972521) 00000100:00000010:3.0:1788972525.849112:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788972525.849132:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 00000100:00000010:3.0:1788972525.849161:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788972525.849166:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 00000100:00000010:3.0:1788972525.849171:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788972525.849175:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 02000000:00000010:3.0:1788972525.849203:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d2fc4400. 02000000:00000010:2.0:1788972525.849340:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624000. 00000400:00000200:2.0:1788972525.849348:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972525.849352:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126420. 00000400:00000010:2.0:1788972525.849358:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972525.849365:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617408, portal 25 00000400:00000010:2.0:1788972525.849370:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972525.849373:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189617408, offset 0 00000400:00000200:2.0:1788972525.849382:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972525.849390:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.849398:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972525.849402:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972525.849406:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972525.849410:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972525.849413:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972525.849416:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [815:815:256:4294967295] 192.168.203.150@tcp LPNI seq info [815:815:8:4294967295] 00000400:00000200:2.0:1788972525.849420:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.849429:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972525.849434:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972525.849440:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387200. 00000800:00000200:2.0:1788972525.849445:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972525.849451:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972525.849455:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972525.849468:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972525.849472:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78300. 00000100:00000010:2.0:1788972525.849553:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78f60. 02000000:00000010:2.0:1788972525.849566:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624c00. 00000400:00000200:2.0:1788972525.849569:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972525.849572:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126a50. 00000400:00000010:2.0:1788972525.849575:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972525.849589:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617664, portal 10 00000400:00000010:2.0:1788972525.849594:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972525.849596:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189617664, offset 0 00000400:00000200:2.0:1788972525.849601:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972525.849604:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.849608:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972525.849611:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972525.849613:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972525.849615:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972525.849617:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972525.849620:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [816:816:255:4294967295] 192.168.203.150@tcp LPNI seq info [816:816:7:4294967295] 00000400:00000200:2.0:1788972525.849623:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.849628:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972525.849632:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972525.849635:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387500. 00000800:00000200:2.0:1788972525.849639:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972525.849643:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972525.849645:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972525.849650:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972525.849690:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972525.849693:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387200. 00000400:00000200:2.0:1788972525.849698:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972525.849702:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972525.849707:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972525.849709:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972525.849717:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790380 x1875872189617408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972541 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972525.850038:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972525.850042:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387500. 00000400:00000200:2.0:1788972525.850046:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972525.850050:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972525.850059:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788972525.850061:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972525.850067:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189617664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972525.850172:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972525.850177:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436840. 00000400:00000010:3.0:1788972525.850187:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972525.850190:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617536, portal 10 00000400:00000010:3.0:1788972525.850195:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972525.850198:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189617536, offset 0 00000400:00000200:3.0:1788972525.850203:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972525.850263:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972525.850269:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972525.850272:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972525.850275:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972525.850277:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972525.850279:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972525.850282:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [817:817:256:4294967295] 192.168.203.150@tcp LPNI seq info [817:817:8:4294967295] 00000400:00000200:3.0:1788972525.850285:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972525.850291:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972525.850296:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972525.850300:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bd00. 00000800:00000200:3.0:1788972525.850304:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972525.850309:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972525.850312:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972525.850330:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972525.850702:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972525.850706:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bd00. 00000400:00000200:2.0:1788972525.850710:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972525.850714:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972525.850718:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:2.0:1788972525.850720:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:2.0:1788972525.850726:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792680 x1875872189617536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972525.853160:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853169:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972525.853177:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853182:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853187:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853190:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819500 00000400:00000200:2.0:1788972525.853195:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853201:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2825 [1] + 224 00000400:00000200:2.0:1788972525.853266:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972525.853272:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972525.853285:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790380 x1875872189617408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972525.853322:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853326:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972525.853331:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853333:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853336:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853339:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819600 00000400:00000200:2.0:1788972525.853355:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853360:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2835 [1] + 224 00000400:00000200:2.0:1788972525.853367:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972525.853370:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972525.853377:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189617664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972525.853390:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853394:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972525.853397:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853400:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853402:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853405:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819580 00000400:00000200:2.0:1788972525.853408:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972525.853412:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2845 [1] + 224 00000400:00000200:2.0:1788972525.853419:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972525.853422:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972525.853428:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792680 x1875872189617536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972525.853563:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436840. 00000400:00000200:3.0:1788972525.853568:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:3.0:1788972525.853570:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972525.853576:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792680 x1875872189617536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972525.853584:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792680 x1875872189617536/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972525.853613:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d2fc4400. 02000000:00000010:3.0:1788972525.853618:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:3.0:1788972525.853623:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792680. 00000400:00000010:3.0:1788972525.853647:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126420. 00000400:00000200:3.0:1788972525.853650:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:3.0:1788972525.853652:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:3.0:1788972525.853657:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790380 x1875872189617408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972525.853665:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790380 x1875872189617408/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972525.853677:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126a50. 00000400:00000200:3.0:1788972525.853679:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:3.0:1788972525.853681:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:3.0:1788972525.853684:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189617664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972525.853692:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189617664/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972541 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972525.853703:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624000. 02000000:00000010:3.0:1788972525.853707:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:3.0:1788972525.853709:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788972525.853711:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624c00. 02000000:00000010:3.0:1788972525.853714:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:3.0:1788972525.853716:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 00000100:00000010:2.0:1788972526.866374:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78300. 00000100:00000010:3.0:1788972526.867023:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78f60. 00000001:02000400:0.0:1788972529.315250:0:128526:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 604: lnet_legacy2yaml: default hop with explicit value ========================================================== 12:48:48 (1788972528) 00000100:00000010:3.0:1788972530.961911:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788972530.961931:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 00000100:00000010:3.0:1788972530.961960:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788972530.961964:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 00000100:00000010:3.0:1788972530.961970:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788972530.961973:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b400. 02000000:00000010:3.0:1788972530.962004:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3a49e00. 00000400:00000200:3.0:1788972530.962014:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972530.962019:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436840. 00000400:00000010:3.0:1788972530.962044:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48734c8. 00000100:00000200:3.0:1788972530.962051:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617792, portal 25 00000400:00000010:3.0:1788972530.962057:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972530.962059:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189617792, offset 0 00000400:00000200:3.0:1788972530.962068:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972530.962077:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972530.962085:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972530.962090:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972530.962094:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972530.962098:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972530.962101:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972530.962104:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [818:818:256:4294967295] 192.168.203.150@tcp LPNI seq info [818:818:8:4294967295] 00000400:00000200:3.0:1788972530.962109:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972530.962119:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972530.962126:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972530.962132:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b100. 00000800:00000200:3.0:1788972530.962138:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972530.962145:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972530.962149:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972530.962163:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972530.962168:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654de40. 00000100:00000010:3.0:1788972530.962179:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d240. 02000000:00000010:3.0:1788972530.962189:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3a49000. 00000400:00000200:3.0:1788972530.962193:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972530.962195:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436cb8. 00000400:00000010:3.0:1788972530.962199:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873b28. 00000100:00000200:3.0:1788972530.962201:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618048, portal 10 00000400:00000010:3.0:1788972530.962207:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788972530.962209:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189618048, offset 0 00000400:00000200:3.0:1788972530.962213:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972530.962217:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972530.962226:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972530.962229:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972530.962231:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972530.962234:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972530.962236:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972530.962238:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [819:819:255:4294967295] 192.168.203.150@tcp LPNI seq info [819:819:7:4294967295] 00000400:00000200:3.0:1788972530.962242:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972530.962248:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972530.962253:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972530.962256:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b700. 00000800:00000200:3.0:1788972530.962260:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972530.962264:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972530.962266:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972530.962272:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972530.962287:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3a49200. 00000400:00000200:3.0:1788972530.962289:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972530.962291:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972530.962294:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8. 00000100:00000200:3.0:1788972530.962297:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189617920, portal 10 00000400:00000010:3.0:1788972530.962300:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873550. 00000100:00000200:3.0:1788972530.962302:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189617920, offset 0 00000400:00000200:3.0:1788972530.962305:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972530.962308:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972530.962311:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972530.962313:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972530.962315:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972530.962317:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972530.962319:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972530.962322:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [820:820:254:4294967295] 192.168.203.150@tcp LPNI seq info [820:820:6:4294967295] 00000400:00000200:3.0:1788972530.962326:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972530.962331:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972530.962341:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972530.962344:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bd00. 00000800:00000200:3.0:1788972530.962347:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972530.962351:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972530.962353:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972530.962358:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972530.962405:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972530.962409:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b100. 00000400:00000200:3.0:1788972530.962414:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972530.962418:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972530.962423:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:3.0:1788972530.962426:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972530.962434:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189617792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972530.962457:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972530.962459:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b700. 00000400:00000200:3.0:1788972530.962462:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972530.962465:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972530.962467:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770 00000400:00000010:3.0:1788972530.962469:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788972530.962473:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792680 x1875872189618048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972530.962545:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972530.962548:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bd00. 00000400:00000200:3.0:1788972530.962551:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972530.962554:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972530.962556:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873550 00000400:00000010:3.0:1788972530.962558:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873550. 00000100:00000200:3.0:1788972530.962562:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790380 x1875872189617920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972530.974121:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974132:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972530.974142:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974147:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974152:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974157:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819780 00000400:00000200:2.0:1788972530.974163:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974168:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2865 [1] + 224 00000400:00000200:2.0:1788972530.974180:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972530.974184:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972530.974214:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792680 x1875872189618048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972530.974249:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974253:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972530.974257:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974260:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974264:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974266:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819700 00000400:00000200:2.0:1788972530.974269:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974273:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2875 [1] + 224 00000400:00000200:2.0:1788972530.974279:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972530.974282:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972530.974287:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790380 x1875872189617920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972530.974299:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974302:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972530.974305:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974307:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974310:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974313:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819680 00000400:00000200:2.0:1788972530.974315:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972530.974319:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2855 [1] + 224 00000400:00000200:2.0:1788972530.974330:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972530.974333:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972530.974337:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189617792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972530.974368:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:2.0:1788972530.974373:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8 00000400:00000010:2.0:1788972530.974375:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8. 00000100:00000200:2.0:1788972530.974387:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790380 x1875872189617920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972530.974396:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790380 x1875872189617920/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972530.974430:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436840. 00000400:00000200:2.0:1788972530.974434:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48734c8 00000400:00000010:2.0:1788972530.974435:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48734c8. 00000100:00000200:2.0:1788972530.974440:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189617792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972530.974447:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189617792/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972530.974458:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436cb8. 00000400:00000200:2.0:1788972530.974461:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873b28 00000400:00000010:2.0:1788972530.974462:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873b28. 00000100:00000200:2.0:1788972530.974466:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792680 x1875872189618048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972530.974473:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792680 x1875872189618048/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972546 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972530.974487:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3a49200. 02000000:00000010:2.0:1788972530.974494:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:2.0:1788972530.974499:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380. 02000000:00000010:2.0:1788972530.974502:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3a49e00. 02000000:00000010:2.0:1788972530.974505:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:2.0:1788972530.974507:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 02000000:00000010:2.0:1788972530.974509:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3a49000. 02000000:00000010:2.0:1788972530.974511:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b400. 00000100:00000010:2.0:1788972530.974512:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792680. 00000100:00000010:3.0:1788972531.986840:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d240. 00000100:00000010:2.0:1788972531.988108:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654de40. 00000100:00000010:2.0:1788972536.090515:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972536.090556:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387100. 00000100:00000010:2.0:1788972536.090594:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972536.090599:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387500. 00000100:00000010:2.0:1788972536.090607:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972536.090611:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387200. 02000000:00000010:2.0:1788972536.090655:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624600. 00000400:00000200:2.0:1788972536.090666:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972536.090671:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126f20. 00000400:00000010:2.0:1788972536.090680:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972536.090689:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618176, portal 25 00000400:00000010:2.0:1788972536.090696:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972536.090699:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189618176, offset 0 00000400:00000200:2.0:1788972536.090711:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972536.090722:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972536.090733:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972536.090740:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972536.090745:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972536.090749:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972536.090753:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972536.090757:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [821:821:256:4294967295] 192.168.203.150@tcp LPNI seq info [821:821:8:4294967295] 00000400:00000200:2.0:1788972536.090762:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972536.090773:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972536.090781:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972536.090786:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387800. 00000800:00000200:2.0:1788972536.090795:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972536.090805:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972536.091281:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972536.091303:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972536.091311:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78360. 00000100:00000010:2.0:1788972536.091328:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78660. 02000000:00000010:2.0:1788972536.091343:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624e00. 00000400:00000200:2.0:1788972536.091354:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972536.091357:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126948. 00000400:00000010:2.0:1788972536.091362:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bdd0. 00000100:00000200:2.0:1788972536.091366:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618432, portal 10 00000400:00000010:2.0:1788972536.091373:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972536.091376:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189618432, offset 0 00000400:00000200:2.0:1788972536.091382:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972536.091387:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972536.091391:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972536.091394:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972536.091397:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972536.091399:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972536.091402:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972536.091406:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [822:822:255:4294967295] 192.168.203.150@tcp LPNI seq info [822:822:7:4294967295] 00000400:00000200:2.0:1788972536.091410:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972536.091417:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972536.091422:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972536.091427:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387d00. 00000800:00000200:2.0:1788972536.091431:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972536.091436:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972536.091439:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972536.091446:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972536.091468:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625c00. 00000400:00000200:2.0:1788972536.091472:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972536.091474:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126318. 00000400:00000010:2.0:1788972536.091478:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b330. 00000100:00000200:2.0:1788972536.091481:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618304, portal 10 00000400:00000010:2.0:1788972536.091485:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972536.091487:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189618304, offset 0 00000400:00000200:2.0:1788972536.091491:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972536.091495:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972536.091504:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972536.091507:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972536.091510:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972536.091512:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972536.091515:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972536.091517:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [823:823:254:4294967295] 192.168.203.150@tcp LPNI seq info [823:823:6:4294967295] 00000400:00000200:2.0:1788972536.091521:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972536.091526:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972536.091531:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972536.091535:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387900. 00000800:00000200:2.0:1788972536.091539:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972536.091543:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972536.091546:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972536.091552:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972536.091618:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972536.091624:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387800. 00000400:00000200:2.0:1788972536.091631:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972536.091636:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972536.091644:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972536.091647:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972536.091658:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189618176/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972536.091687:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972536.091689:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387d00. 00000400:00000200:2.0:1788972536.091692:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972536.091696:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972536.091698:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972536.091700:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972536.091704:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189618432/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972536.091796:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972536.091805:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387900. 00000400:00000200:2.0:1788972536.091808:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972536.091840:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972536.091844:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788972536.091845:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972536.091851:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189618304/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972536.095431:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095443:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972536.095454:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095460:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095466:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095471:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819900 00000400:00000200:3.0:1788972536.095478:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095485:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2895 [1] + 224 00000400:00000200:3.0:1788972536.095500:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972536.095505:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972536.095523:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189618432/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972536.095559:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095563:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972536.095567:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095569:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095571:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095574:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819880 00000400:00000200:3.0:1788972536.095577:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.095580:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x28a5 [1] + 224 00000400:00000200:3.0:1788972536.095589:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972536.095592:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972536.095598:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189618304/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972536.095629:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126318. 00000400:00000200:3.0:1788972536.095633:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b330 00000400:00000010:3.0:1788972536.095636:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b330. 00000100:00000200:3.0:1788972536.095642:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189618304/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972536.095650:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189618304/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972536.095696:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126948. 00000400:00000200:3.0:1788972536.095700:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bdd0 00000400:00000010:3.0:1788972536.095701:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bdd0. 00000100:00000200:3.0:1788972536.095706:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189618432/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972536.095713:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189618432/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972536.095730:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625c00. 02000000:00000010:3.0:1788972536.095738:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387500. 00000100:00000010:3.0:1788972536.095744:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 02000000:00000010:3.0:1788972536.095748:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624e00. 02000000:00000010:3.0:1788972536.095751:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387200. 00000100:00000010:3.0:1788972536.095754:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 00000400:00000200:3.0:1788972536.096316:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.096321:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972536.096326:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.096329:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.096333:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.096336:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819800 00000400:00000200:3.0:1788972536.096340:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972536.096344:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2885 [1] + 224 00000400:00000200:3.0:1788972536.096354:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972536.096358:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972536.096366:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189618176/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972536.096398:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126f20. 00000400:00000200:3.0:1788972536.096402:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:3.0:1788972536.096404:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:3.0:1788972536.096464:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189618176/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972536.096481:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189618176/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972552 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972536.096498:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624600. 02000000:00000010:3.0:1788972536.096502:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387100. 00000100:00000010:3.0:1788972536.096505:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 00000100:00000010:2.0:1788972537.105915:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78360. 00000100:00000010:3.0:1788972537.106260:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78660. 00000001:02000400:3.0:1788972537.845771:0:129106:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 605: lnet_legacy2yaml: quote leading star in IP pattern ========================================================== 12:48:56 (1788972536) 00000100:00000010:0.0:1788972541.207930:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972541.207961:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7600. 00000100:00000010:0.0:1788972541.208005:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972541.208012:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972541.208019:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972541.208023:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7500. 02000000:00000010:0.0:1788972541.208070:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000400:00000200:0.0:1788972541.208083:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972541.208088:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4210. 00000400:00000010:0.0:1788972541.208096:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788972541.208476:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618560, portal 25 00000400:00000010:0.0:1788972541.208484:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788972541.208489:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189618560, offset 0 00000400:00000200:0.0:1788972541.208500:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972541.208511:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972541.208521:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972541.208527:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972541.208541:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972541.208545:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972541.208549:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972541.208553:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [824:824:256:4294967295] 192.168.203.150@tcp LPNI seq info [824:824:8:4294967295] 00000400:00000200:0.0:1788972541.208558:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972541.208569:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972541.208576:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972541.208583:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0100. 00000800:00000200:0.0:1788972541.208591:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972541.208600:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972541.208605:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972541.208658:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972541.208664:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49360. 00000100:00000010:0.0:1788972541.208676:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49060. 02000000:00000010:0.0:1788972541.208687:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b200. 00000400:00000200:0.0:1788972541.208691:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972541.208694:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972541.208697:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972541.208700:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618816, portal 10 00000400:00000010:0.0:1788972541.208707:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972541.208709:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189618816, offset 0 00000400:00000200:0.0:1788972541.208713:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972541.208717:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972541.208720:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972541.208723:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972541.208725:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972541.208728:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972541.208730:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972541.208733:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [825:825:255:4294967295] 192.168.203.150@tcp LPNI seq info [825:825:7:4294967295] 00000400:00000200:0.0:1788972541.208737:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972541.208743:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972541.208753:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972541.208756:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0c00. 00000800:00000200:0.0:1788972541.208760:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972541.208764:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972541.208767:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972541.208773:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972541.208792:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5a9a000. 00000400:00000200:0.0:1788972541.208795:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972541.208797:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4d68. 00000400:00000010:0.0:1788972541.208800:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972541.208803:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618688, portal 10 00000400:00000010:0.0:1788972541.208808:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8f68. 00000100:00000200:0.0:1788972541.208810:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189618688, offset 0 00000400:00000200:0.0:1788972541.208844:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972541.208847:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972541.208851:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972541.208853:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972541.208855:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972541.208858:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972541.208860:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972541.208863:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [826:826:254:4294967295] 192.168.203.150@tcp LPNI seq info [826:826:6:4294967295] 00000400:00000200:0.0:1788972541.208867:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972541.208874:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972541.208878:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972541.208881:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0200. 00000800:00000200:0.0:1788972541.208885:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972541.208889:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972541.208892:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972541.208896:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972541.211717:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972541.211729:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0100. 00000400:00000200:3.0:1788972541.211739:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972541.211749:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972541.211759:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:3.0:1788972541.211763:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:3.0:1788972541.211778:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189618560/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972541.211809:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972541.211840:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0c00. 00000400:00000200:3.0:1788972541.211845:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972541.211849:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972541.211852:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880 00000400:00000010:3.0:1788972541.211853:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880. 00000100:00000200:3.0:1788972541.211859:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189618816/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972541.211944:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972541.211947:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0200. 00000400:00000200:3.0:1788972541.211951:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972541.211954:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972541.211957:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8f68 00000400:00000010:3.0:1788972541.211959:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8f68. 00000100:00000200:3.0:1788972541.211974:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189618688/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972541.215862:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.215874:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972541.215885:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.215891:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.215897:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.215901:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819a80 00000400:00000200:3.0:1788972541.215908:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.215915:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x28bd [1] + 224 00000400:00000200:3.0:1788972541.215927:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972541.215932:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972541.216155:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189618816/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972541.216194:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.216197:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972541.216201:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.216204:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.216206:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.216209:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819a00 00000400:00000200:3.0:1788972541.216213:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.216216:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x28c5 [1] + 224 00000400:00000200:3.0:1788972541.216226:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972541.216229:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972541.216236:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189618688/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972541.216799:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.216807:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972541.217075:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.217094:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.217098:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.217102:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819980 00000400:00000200:3.0:1788972541.217107:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972541.217113:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x28b5 [1] + 224 00000400:00000200:3.0:1788972541.217124:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972541.217128:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972541.217136:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189618560/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972541.217455:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4d68. 00000400:00000200:0.0:1788972541.217467:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550 00000400:00000010:0.0:1788972541.217470:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972541.217478:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189618688/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972541.217489:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189618688/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972541.217526:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4210. 00000400:00000200:0.0:1788972541.217530:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38 00000400:00000010:0.0:1788972541.217532:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788972541.217537:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189618560/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972541.217546:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189618560/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972541.217559:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972541.217562:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28 00000400:00000010:0.0:1788972541.217564:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972541.217569:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189618816/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972541.217578:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189618816/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972541.217597:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5a9a000. 02000000:00000010:0.0:1788972541.217606:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972541.217612:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972541.217616:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:0.0:1788972541.217620:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7600. 00000100:00000010:0.0:1788972541.217622:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972541.217624:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b200. 02000000:00000010:0.0:1788972541.217626:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7500. 00000100:00000010:0.0:1788972541.217628:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000100:00000010:2.0:1788972542.226895:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49360. 00000100:00000010:3.0:1788972542.227247:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49060. 00000001:02000400:3.0:1788972544.834133:0:129688:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 606: lnet_legacy2yaml: file inputs === 12:49:03 (1788972543) 00000100:00000010:0.0:1788972546.321923:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972546.321950:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0200. 00000100:00000010:0.0:1788972546.321979:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972546.321985:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0c00. 00000100:00000010:0.0:1788972546.321991:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972546.321995:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0400. 02000000:00000010:0.0:1788972546.322028:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d408da00. 00000400:00000200:0.0:1788972546.322038:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972546.322043:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972546.322050:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972546.322057:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189618944, portal 25 00000400:00000010:0.0:1788972546.322062:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788972546.322065:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189618944, offset 0 00000400:00000200:0.0:1788972546.322075:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972546.322083:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972546.322090:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972546.322095:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972546.322099:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972546.322104:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972546.322106:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972546.322109:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [827:827:256:4294967295] 192.168.203.150@tcp LPNI seq info [827:827:8:4294967295] 00000400:00000200:0.0:1788972546.322114:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972546.322123:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972546.322128:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972546.322133:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0100. 00000800:00000200:0.0:1788972546.322139:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972546.322146:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972546.322151:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972546.322163:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972546.322168:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e494e0. 00000100:00000010:0.0:1788972546.322178:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49ba0. 02000000:00000010:0.0:1788972546.322189:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d408d400. 00000400:00000200:0.0:1788972546.322199:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972546.322201:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4210. 00000400:00000010:0.0:1788972546.322205:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972546.322208:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619200, portal 10 00000400:00000010:0.0:1788972546.322213:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8110. 00000100:00000200:0.0:1788972546.322216:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189619200, offset 0 00000400:00000200:0.0:1788972546.322220:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972546.322224:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972546.322228:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972546.322230:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972546.322233:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972546.322235:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972546.322237:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972546.322240:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [828:828:255:4294967295] 192.168.203.150@tcp LPNI seq info [828:828:7:4294967295] 00000400:00000200:0.0:1788972546.322244:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972546.322249:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972546.322253:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972546.322258:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7d00. 00000800:00000200:0.0:1788972546.322262:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972546.322267:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972546.322270:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972546.322276:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972546.322295:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d932c800. 00000400:00000200:0.0:1788972546.322298:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972546.322301:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4d68. 00000400:00000010:0.0:1788972546.322304:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972546.322307:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619072, portal 10 00000400:00000010:0.0:1788972546.322311:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a85d8. 00000100:00000200:0.0:1788972546.322314:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189619072, offset 0 00000400:00000200:0.0:1788972546.322318:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972546.322321:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972546.322331:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972546.322333:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972546.322336:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972546.322339:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972546.322341:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972546.322344:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [829:829:254:4294967295] 192.168.203.150@tcp LPNI seq info [829:829:6:4294967295] 00000400:00000200:0.0:1788972546.322348:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972546.322354:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972546.322359:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972546.322363:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7500. 00000800:00000200:0.0:1788972546.322367:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972546.322371:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972546.322374:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972546.322380:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972546.322534:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972546.322539:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0100. 00000400:00000200:3.0:1788972546.322546:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972546.322551:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972546.322556:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38 00000400:00000010:3.0:1788972546.322558:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38. 00000100:00000200:3.0:1788972546.322567:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189618944/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972546.322592:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972546.322595:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7d00. 00000400:00000200:3.0:1788972546.322597:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972546.322600:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972546.322602:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8110 00000400:00000010:3.0:1788972546.322604:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8110. 00000100:00000200:3.0:1788972546.322609:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189619200/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972546.322672:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972546.322675:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7500. 00000400:00000200:3.0:1788972546.322678:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972546.322681:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972546.322690:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a85d8 00000400:00000010:3.0:1788972546.322692:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a85d8. 00000100:00000200:3.0:1788972546.322697:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189619072/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972546.330774:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330795:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972546.330805:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330810:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330839:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330843:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819c00 00000400:00000200:2.0:1788972546.330849:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330855:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x28d5 [1] + 224 00000400:00000200:2.0:1788972546.330867:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972546.330872:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972546.330886:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189619200/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972546.330918:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330921:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972546.330925:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330928:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330930:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330933:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819b80 00000400:00000200:2.0:1788972546.330936:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.330940:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x28dd [1] + 224 00000400:00000200:2.0:1788972546.330949:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972546.330960:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972546.330967:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189619072/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972546.335105:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4d68. 00000400:00000200:0.0:1788972546.335116:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a87f8 00000400:00000010:0.0:1788972546.335119:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972546.335132:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189619072/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972546.335144:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189619072/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972546.335173:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4210. 00000400:00000200:0.0:1788972546.335176:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550 00000400:00000010:0.0:1788972546.335177:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972546.335182:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189619200/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972546.335190:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189619200/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972546.335204:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d932c800. 02000000:00000010:0.0:1788972546.335220:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0c00. 00000100:00000010:0.0:1788972546.335227:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972546.335231:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d408d400. 02000000:00000010:0.0:1788972546.335235:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0400. 00000100:00000010:0.0:1788972546.335237:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 00000400:00000200:2.0:1788972546.335551:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.335557:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972546.335564:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.335569:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.335574:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.335578:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819b00 00000400:00000200:2.0:1788972546.335583:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972546.335588:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x28cd [1] + 224 00000400:00000200:2.0:1788972546.335602:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972546.335607:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972546.335616:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189618944/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972546.337431:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972546.337440:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28 00000400:00000010:0.0:1788972546.337443:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972546.337454:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189618944/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972546.337464:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189618944/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972546.337491:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d408da00. 02000000:00000010:0.0:1788972546.337497:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0200. 00000100:00000010:0.0:1788972546.337501:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000100:00000010:2.0:1788972547.348184:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e494e0. 00000100:00000010:3.0:1788972547.350122:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49ba0. 00000100:00000010:2.0:1788972551.443375:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c380. 02000000:00000010:2.0:1788972551.443401:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d4603900. 00000100:00000010:2.0:1788972551.443446:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972551.443452:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d4603d00. 00000100:00000010:2.0:1788972551.443459:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972551.443464:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d4603300. 02000000:00000010:2.0:1788972551.443494:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc4a00. 00000400:00000200:2.0:1788972551.443504:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972551.443517:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580. 00000400:00000010:2.0:1788972551.443525:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972551.443532:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619456, portal 10 00000400:00000010:2.0:1788972551.443539:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972551.443542:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189619456, offset 0 00000400:00000200:2.0:1788972551.443550:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972551.443568:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.443576:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972551.443581:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972551.443585:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972551.443589:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972551.443591:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972551.443595:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [830:830:256:4294967295] 192.168.203.150@tcp LPNI seq info [830:830:8:4294967295] 00000400:00000200:2.0:1788972551.443599:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.443608:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972551.443613:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972551.443618:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d4603800. 00000800:00000200:2.0:1788972551.443624:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972551.443631:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972551.443635:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d4603800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972551.443649:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972551.443653:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78ae0. 00000100:00000010:2.0:1788972551.443663:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78de0. 02000000:00000010:2.0:1788972551.443687:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5600. 00000400:00000200:2.0:1788972551.443691:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972551.443694:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126e70. 00000400:00000010:2.0:1788972551.443697:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972551.443700:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619328, portal 25 00000400:00000010:2.0:1788972551.443703:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972551.443705:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189619328, offset 0 00000400:00000200:2.0:1788972551.443709:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972551.443712:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.443715:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972551.443717:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972551.443719:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972551.443721:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972551.443723:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972551.443732:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [831:831:255:4294967295] 192.168.203.150@tcp LPNI seq info [831:831:7:4294967295] 00000400:00000200:2.0:1788972551.443736:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.443742:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972551.443756:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972551.443760:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387100. 00000800:00000200:2.0:1788972551.443764:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972551.443769:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972551.443771:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972551.443776:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972551.443788:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c5dc5e00. 00000400:00000200:2.0:1788972551.443792:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972551.443795:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972551.443798:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972551.443801:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619584, portal 10 00000400:00000010:2.0:1788972551.443805:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972551.443807:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189619584, offset 0 00000400:00000200:2.0:1788972551.443812:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972551.443847:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.443851:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972551.443853:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972551.443855:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972551.443858:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972551.443860:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972551.443862:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [832:832:254:4294967295] 192.168.203.150@tcp LPNI seq info [832:832:6:4294967295] 00000400:00000200:2.0:1788972551.443866:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.443872:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972551.443877:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972551.443881:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387e00. 00000800:00000200:2.0:1788972551.443884:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972551.443888:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972551.443898:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972551.443904:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972551.443950:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972551.443955:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d4603800. 00000400:00000200:2.0:1788972551.443961:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972551.443966:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972551.443973:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972551.443975:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972551.443984:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189619456/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972551.444009:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972551.444012:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387100. 00000400:00000200:2.0:1788972551.444015:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972551.444019:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972551.444022:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972551.444023:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972551.444028:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c380 x1875872189619328/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972567 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972551.444103:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972551.444106:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387e00. 00000400:00000200:2.0:1788972551.444110:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972551.444113:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972551.444116:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972551.444117:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972551.444122:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189619584/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972551.447718:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.447729:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972551.447737:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.447741:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.447758:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.447769:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819c80 00000400:00000200:2.0:1788972551.447775:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972551.447781:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x28f5 [1] + 224 00000400:00000200:2.0:1788972551.447795:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972551.447799:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972551.447812:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c380 x1875872189619328/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972551.447990:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126e70. 00000400:00000200:3.0:1788972551.447995:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:3.0:1788972551.447998:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:3.0:1788972551.448005:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c380 x1875872189619328/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972551.448014:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c380 x1875872189619328/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972551.448042:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5600. 02000000:00000010:3.0:1788972551.448047:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d4603900. 00000100:00000010:3.0:1788972551.448052:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c380. 00000400:00000200:3.0:1788972551.450622:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.450631:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972551.450641:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.450645:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.450650:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.450654:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819d00 00000400:00000200:3.0:1788972551.450659:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.450666:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x28e5 [1] + 224 00000400:00000200:3.0:1788972551.451553:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972551.451563:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972551.451578:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189619456/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972551.451613:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.451617:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972551.451622:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.451633:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.451636:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.451640:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819d80 00000400:00000200:3.0:1788972551.451643:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972551.451648:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2905 [1] + 224 00000400:00000200:3.0:1788972551.451659:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972551.451662:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972551.451669:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189619584/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972551.453185:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580. 00000400:00000200:2.0:1788972551.453204:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788972551.453207:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972551.453219:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189619456/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972551.453237:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189619456/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972551.453279:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc4a00. 02000000:00000010:2.0:1788972551.453286:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d4603d00. 00000100:00000010:2.0:1788972551.453290:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 00000400:00000010:2.0:1788972551.453975:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:2.0:1788972551.454167:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788972551.454170:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972551.454180:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189619584/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972551.454190:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189619584/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972567 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972551.454209:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c5dc5e00. 02000000:00000010:2.0:1788972551.454214:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d4603300. 00000100:00000010:2.0:1788972551.454217:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000100:00000010:2.0:1788972552.467718:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78ae0. 00000100:00000010:3.0:1788972552.471548:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78de0. 00000001:02000400:3.0:1788972552.768748:0:130269:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 607: lnet_legacy2yaml: options line with combined ip2nets and routes ========================================================== 12:49:11 (1788972551) 00000100:00000010:3.0:1788972556.564857:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791180. 02000000:00000010:3.0:1788972556.564886:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b300. 00000100:00000010:3.0:1788972556.564921:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791f80. 02000000:00000010:3.0:1788972556.564926:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b700. 00000100:00000010:3.0:1788972556.564933:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790700. 02000000:00000010:3.0:1788972556.564937:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1be00. 02000000:00000010:3.0:1788972556.565129:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946600. 00000400:00000200:3.0:1788972556.565143:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972556.565159:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788972556.565176:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873550. 00000100:00000200:3.0:1788972556.565185:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619840, portal 10 00000400:00000010:3.0:1788972556.565194:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873770. 00000100:00000200:3.0:1788972556.565198:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189619840, offset 0 00000400:00000200:3.0:1788972556.565209:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972556.565221:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.565231:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972556.565238:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972556.565242:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972556.565247:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972556.565265:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972556.565270:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [833:833:256:4294967295] 192.168.203.150@tcp LPNI seq info [833:833:8:4294967295] 00000400:00000200:3.0:1788972556.565275:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.565287:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972556.565295:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972556.565302:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bd00. 00000800:00000200:3.0:1788972556.565311:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972556.565320:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972556.565326:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972556.565346:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972556.565352:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654dcc0. 00000100:00000010:3.0:1788972556.565365:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d2a0. 02000000:00000010:3.0:1788972556.565389:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946000. 00000400:00000200:3.0:1788972556.565393:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972556.565396:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68. 00000400:00000010:3.0:1788972556.565400:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972556.565403:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619712, portal 25 00000400:00000010:3.0:1788972556.565407:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788972556.565409:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189619712, offset 0 00000400:00000200:3.0:1788972556.565413:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972556.565417:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.565420:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972556.565422:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972556.565425:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972556.565427:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972556.565429:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972556.565432:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [834:834:255:4294967295] 192.168.203.150@tcp LPNI seq info [834:834:7:4294967295] 00000400:00000200:3.0:1788972556.565436:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.565442:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972556.565446:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972556.565455:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bb00. 00000800:00000200:3.0:1788972556.565460:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972556.565465:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972556.565467:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972556.565472:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972556.565484:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0946e00. 00000400:00000200:3.0:1788972556.565488:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972556.565491:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688. 00000400:00000010:3.0:1788972556.565495:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788972556.565498:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189619968, portal 10 00000400:00000010:3.0:1788972556.565502:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972556.565504:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189619968, offset 0 00000400:00000200:3.0:1788972556.565509:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972556.565512:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.565516:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972556.565519:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972556.565521:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972556.565524:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972556.565525:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972556.565529:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [835:835:254:4294967295] 192.168.203.150@tcp LPNI seq info [835:835:6:4294967295] 00000400:00000200:3.0:1788972556.565532:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.565538:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972556.565542:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972556.565551:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b000. 00000800:00000200:3.0:1788972556.565555:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972556.565560:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972556.565562:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972556.565567:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972556.566349:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972556.566367:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bd00. 00000400:00000200:2.0:1788972556.566376:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972556.566383:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972556.566390:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873770 00000400:00000010:2.0:1788972556.566393:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873770. 00000100:00000200:2.0:1788972556.566405:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791f80 x1875872189619840/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972556.566434:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972556.566437:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bb00. 00000400:00000200:2.0:1788972556.566440:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972556.566443:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972556.566446:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8 00000400:00000010:2.0:1788972556.566448:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8. 00000100:00000200:2.0:1788972556.566452:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791180 x1875872189619712/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972556.566539:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972556.566542:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b000. 00000400:00000200:2.0:1788972556.566546:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972556.566549:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972556.566552:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:2.0:1788972556.566554:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:2.0:1788972556.566560:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790700 x1875872189619968/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972556.575408:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575423:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972556.575433:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575457:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575465:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575469:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819e00 00000400:00000200:3.0:1788972556.575730:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575741:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2925 [1] + 224 00000400:00000200:3.0:1788972556.575758:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972556.575764:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972556.575784:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791180 x1875872189619712/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972556.575849:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575873:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972556.575879:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575882:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575885:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575888:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819e80 00000400:00000200:3.0:1788972556.575891:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575896:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2915 [1] + 224 00000400:00000200:3.0:1788972556.575903:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972556.575906:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972556.575913:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791f80 x1875872189619840/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972556.575927:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575930:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972556.575934:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575936:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575939:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575941:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887819f00 00000400:00000200:3.0:1788972556.575944:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972556.575948:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2935 [1] + 224 00000400:00000200:3.0:1788972556.575956:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972556.575967:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972556.575976:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790700 x1875872189619968/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972556.576025:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68. 00000400:00000200:2.0:1788972556.576032:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:2.0:1788972556.576035:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:2.0:1788972556.576044:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791180 x1875872189619712/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972556.576055:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791180 x1875872189619712/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972556.576085:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688. 00000400:00000200:2.0:1788972556.576096:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:2.0:1788972556.576098:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:2.0:1788972556.576103:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790700 x1875872189619968/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972556.576112:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790700 x1875872189619968/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972556.576370:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:2.0:1788972556.576375:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873550 00000400:00000010:2.0:1788972556.576377:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873550. 00000100:00000200:2.0:1788972556.576382:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791f80 x1875872189619840/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972556.576393:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791f80 x1875872189619840/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972572 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972556.576413:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946000. 02000000:00000010:2.0:1788972556.576422:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b300. 00000100:00000010:2.0:1788972556.576428:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791180. 02000000:00000010:2.0:1788972556.576433:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946e00. 02000000:00000010:2.0:1788972556.576436:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1be00. 00000100:00000010:2.0:1788972556.576439:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790700. 02000000:00000010:2.0:1788972556.576443:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0946600. 02000000:00000010:2.0:1788972556.576445:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b700. 00000100:00000010:2.0:1788972556.576447:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791f80. 00000100:00000010:2.0:1788972557.587770:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654dcc0. 00000100:00000010:3.0:1788972557.587981:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d2a0. 00000001:02000400:2.0:1788972560.685135:0:130849:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 608: lnet_legacy2yaml: error on invalid rule (missing IP pattern) ========================================================== 12:49:19 (1788972559) 00000100:00000010:1.0:1788972561.685243:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800. 02000000:00000010:1.0:1788972561.685264:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d900. 00000100:00000010:1.0:1788972561.685300:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788972561.685305:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d500. 00000100:00000010:1.0:1788972561.685311:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80. 02000000:00000010:1.0:1788972561.685314:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d100. 02000000:00000010:1.0:1788972561.685352:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3385200. 00000400:00000200:1.0:1788972561.685377:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972561.685381:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972561.685387:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788972561.685394:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620096, portal 25 00000400:00000010:1.0:1788972561.685399:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788972561.685402:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189620096, offset 0 00000400:00000200:1.0:1788972561.685411:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972561.685419:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972561.685426:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972561.685431:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972561.685435:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972561.685438:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972561.685440:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972561.685444:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [836:836:256:4294967295] 192.168.203.150@tcp LPNI seq info [836:836:8:4294967295] 00000400:00000200:1.0:1788972561.685448:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972561.685458:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972561.685464:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972561.685468:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d300. 00000800:00000200:1.0:1788972561.685475:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972561.685482:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972561.685487:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972561.685509:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972561.685514:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106a20. 00000100:00000010:1.0:1788972561.685524:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106c00. 02000000:00000010:1.0:1788972561.685534:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3384e00. 00000400:00000200:1.0:1788972561.685537:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972561.685540:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d6e0. 00000400:00000010:1.0:1788972561.685543:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525cc0. 00000100:00000200:1.0:1788972561.685546:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620352, portal 10 00000400:00000010:1.0:1788972561.685551:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525330. 00000100:00000200:1.0:1788972561.685553:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189620352, offset 0 00000400:00000200:1.0:1788972561.685564:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972561.685568:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972561.685571:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972561.685574:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972561.685576:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972561.685578:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972561.685580:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972561.685583:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [837:837:255:4294967295] 192.168.203.150@tcp LPNI seq info [837:837:7:4294967295] 00000400:00000200:1.0:1788972561.685587:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972561.685592:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972561.685597:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972561.685601:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d600. 00000800:00000200:1.0:1788972561.685605:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972561.685609:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972561.685612:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972561.685618:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972561.685649:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3384a00. 00000400:00000200:1.0:1788972561.685653:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972561.685656:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972561.685659:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788972561.685662:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620224, portal 10 00000400:00000010:1.0:1788972561.685666:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525b28. 00000100:00000200:1.0:1788972561.685668:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189620224, offset 0 00000400:00000200:1.0:1788972561.685682:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972561.685686:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972561.685689:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972561.685692:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972561.685694:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972561.685696:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972561.685698:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972561.685701:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [838:838:254:4294967295] 192.168.203.150@tcp LPNI seq info [838:838:6:4294967295] 00000400:00000200:1.0:1788972561.685710:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972561.685716:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972561.685720:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972561.685724:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4cd00. 00000800:00000200:1.0:1788972561.685728:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972561.685731:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972561.685734:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4cd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972561.685740:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972561.685882:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972561.685887:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d300. 00000400:00000200:3.0:1788972561.685894:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972561.685900:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972561.685906:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220 00000400:00000010:3.0:1788972561.685908:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220. 00000100:00000200:3.0:1788972561.685917:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3800 x1875872189620096/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972561.685938:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972561.685940:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d600. 00000400:00000200:3.0:1788972561.685942:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972561.685945:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972561.685947:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525330 00000400:00000010:3.0:1788972561.685949:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525330. 00000100:00000200:3.0:1788972561.685959:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1f80 x1875872189620352/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972561.686018:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972561.686021:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4cd00. 00000400:00000200:3.0:1788972561.686026:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972561.686029:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972561.686032:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525b28 00000400:00000010:3.0:1788972561.686033:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525b28. 00000100:00000200:3.0:1788972561.686038:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189620224/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972561.689646:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689657:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972561.689665:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689689:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689694:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689699:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a080 00000400:00000200:3.0:1788972561.689705:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689711:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x294d [1] + 224 00000400:00000200:3.0:1788972561.689723:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972561.689728:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972561.689741:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1f80 x1875872189620352/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972561.689772:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689776:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972561.689780:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689783:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689786:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689788:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a000 00000400:00000200:3.0:1788972561.689791:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689794:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2955 [1] + 224 00000400:00000200:3.0:1788972561.689800:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972561.689803:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972561.689809:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189620224/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972561.689852:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689856:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972561.689860:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689862:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689864:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689867:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887819f80 00000400:00000200:3.0:1788972561.689871:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972561.689874:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2945 [1] + 224 00000400:00000200:3.0:1788972561.689892:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972561.689895:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972561.689901:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3800 x1875872189620096/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972561.690947:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:0.0:1788972561.690955:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525110 00000400:00000010:0.0:1788972561.690958:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525110. 00000100:00000200:0.0:1788972561.690966:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189620224/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972561.690977:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189620224/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972561.691007:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:0.0:1788972561.691010:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68 00000400:00000010:0.0:1788972561.691012:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68. 00000100:00000200:0.0:1788972561.691016:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3800 x1875872189620096/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972561.691024:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3800 x1875872189620096/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972561.691035:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d6e0. 00000400:00000200:0.0:1788972561.691043:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525cc0 00000400:00000010:0.0:1788972561.691045:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525cc0. 00000100:00000200:0.0:1788972561.691050:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1f80 x1875872189620352/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972561.691058:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1f80 x1875872189620352/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972577 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972561.691077:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3384a00. 02000000:00000010:0.0:1788972561.691083:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d500. 00000100:00000010:0.0:1788972561.691087:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 02000000:00000010:0.0:1788972561.691091:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3385200. 02000000:00000010:0.0:1788972561.691093:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d900. 00000100:00000010:0.0:1788972561.691095:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800. 02000000:00000010:0.0:1788972561.691098:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3384e00. 02000000:00000010:0.0:1788972561.691100:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d100. 00000100:00000010:0.0:1788972561.691102:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80. 00000100:00000010:3.0:1788972562.706233:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106c00. 00000100:00000010:2.0:1788972562.710552:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106a20. 00000100:00000010:1.0:1788972566.801930:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972566.801961:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c600. 00000100:00000010:1.0:1788972566.802000:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972566.802005:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c800. 00000100:00000010:1.0:1788972566.802012:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972566.802015:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c500. 02000000:00000010:1.0:1788972566.802208:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003800. 00000400:00000200:1.0:1788972566.802220:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972566.802225:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1db58. 00000400:00000010:1.0:1788972566.802233:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972566.802241:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620480, portal 25 00000400:00000010:1.0:1788972566.802247:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788972566.802251:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189620480, offset 0 00000400:00000200:1.0:1788972566.802262:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972566.802273:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972566.802282:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972566.802289:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972566.802294:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972566.802299:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972566.802302:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972566.802306:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [839:839:256:4294967295] 192.168.203.150@tcp LPNI seq info [839:839:8:4294967295] 00000400:00000200:1.0:1788972566.802310:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972566.802334:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972566.802341:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972566.802347:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c100. 00000800:00000200:1.0:1788972566.802355:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972566.802365:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972566.802371:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972566.802393:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972566.802399:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106f00. 00000100:00000010:1.0:1788972566.802412:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c41065a0. 02000000:00000010:1.0:1788972566.802427:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003400. 00000400:00000200:1.0:1788972566.802431:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972566.802434:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d2c0. 00000400:00000010:1.0:1788972566.802438:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788972566.802442:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620736, portal 10 00000400:00000010:1.0:1788972566.802448:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000. 00000100:00000200:1.0:1788972566.802451:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189620736, offset 0 00000400:00000200:1.0:1788972566.802456:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972566.802459:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972566.802463:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972566.802466:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972566.802469:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972566.802471:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972566.802473:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972566.802476:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [840:840:255:4294967295] 192.168.203.150@tcp LPNI seq info [840:840:7:4294967295] 00000400:00000200:1.0:1788972566.802480:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972566.802486:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972566.802490:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972566.802494:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4cb00. 00000800:00000200:1.0:1788972566.802498:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972566.802502:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972566.802511:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4cb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972566.802518:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972566.802540:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002400. 00000400:00000200:1.0:1788972566.802544:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972566.802547:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d630. 00000400:00000010:1.0:1788972566.802550:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8. 00000100:00000200:1.0:1788972566.802553:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620608, portal 10 00000400:00000010:1.0:1788972566.802559:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788972566.802562:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189620608, offset 0 00000400:00000200:1.0:1788972566.802566:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972566.802569:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972566.802573:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972566.802576:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972566.802578:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972566.802602:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972566.802604:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972566.802607:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [841:841:254:4294967295] 192.168.203.150@tcp LPNI seq info [841:841:6:4294967295] 00000400:00000200:1.0:1788972566.802611:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972566.802629:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972566.802708:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972566.802714:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c100. 00000400:00000200:3.0:1788972566.802724:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972566.802730:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972566.802738:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:3.0:1788972566.802741:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:3.0:1788972566.802752:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189620480/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972582 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972566.802876:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972566.802879:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4cb00. 00000400:00000200:3.0:1788972566.802883:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972566.802887:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972566.802890:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000 00000400:00000010:3.0:1788972566.802891:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000. 00000100:00000200:3.0:1788972566.802897:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1c00 x1875872189620736/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1788972566.802926:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972566.802930:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4cb00. 00000800:00000200:1.0:1788972566.802935:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972566.802940:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972566.802943:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4cb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972566.802960:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972566.803009:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972566.803012:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4cb00. 00000400:00000200:3.0:1788972566.803015:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972566.803019:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972566.803022:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:3.0:1788972566.803023:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:3.0:1788972566.803029:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189620608/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972566.816169:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816181:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972566.816193:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816198:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816219:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816223:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781a100 00000400:00000200:2.0:1788972566.816230:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816237:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x295d [1] + 224 00000400:00000200:2.0:1788972566.816252:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972566.816426:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972566.816446:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189620480/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972566.816532:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816536:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972566.816541:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816543:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816546:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816550:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a180 00000400:00000200:2.0:1788972566.816553:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.816558:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x296d [1] + 224 00000400:00000200:2.0:1788972566.816570:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972566.816574:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972566.816581:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189620608/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972566.816886:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1db58. 00000400:00000200:1.0:1788972566.816893:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18 00000400:00000010:1.0:1788972566.816896:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972566.816904:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189620480/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972566.816913:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189620480/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972566.816958:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003800. 02000000:00000010:1.0:1788972566.816966:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c600. 00000100:00000010:1.0:1788972566.816974:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000400:00000010:1.0:1788972566.816983:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d630. 00000400:00000200:1.0:1788972566.816985:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8 00000400:00000010:1.0:1788972566.816987:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8. 00000100:00000200:1.0:1788972566.816991:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189620608/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972566.816999:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189620608/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972566.817017:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002400. 02000000:00000010:1.0:1788972566.817021:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c800. 00000100:00000010:1.0:1788972566.817024:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 00000400:00000200:2.0:1788972566.817969:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.818104:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972566.818116:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.818120:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.818125:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.818128:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a200 00000400:00000200:2.0:1788972566.818133:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972566.818140:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2965 [1] + 224 00000400:00000200:2.0:1788972566.818155:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972566.818170:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972566.818183:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1c00 x1875872189620736/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972566.823607:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d2c0. 00000400:00000200:1.0:1788972566.823620:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:1.0:1788972566.823624:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788972566.823650:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1c00 x1875872189620736/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972566.823665:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1c00 x1875872189620736/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972582 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972566.823710:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003400. 02000000:00000010:1.0:1788972566.823718:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c500. 00000100:00000010:1.0:1788972566.823727:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00. 00000400:00000200:2.0:1788972567.110894:0:102136:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:2.0:1788972567.111479:0:102136:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.50@tcp from 255.3.0.2 00000800:00000010:2.0:1788972567.111486:0:102136:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at ffff9a73c7ced760. 00000800:00000010:3.0:1788972567.111645:0:102129:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at ffff9a73c5dce000. 00000800:00000010:3.0:1788972567.111652:0:102129:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff9a73c96019c0. 00000400:00000200:3.0:1788972567.112757:0:102129:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:3.0:1788972567.112762:0:102129:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000800:00000010:3.0:1788972567.112770:0:102129:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff9a73c9601840. 00000800:00000200:3.0:1788972567.112785:0:102129:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.47@tcp p 3.x 192.168.203.50 -> 192.168.203.47:1023 incarnation:1788972565726100512 sched[0] 00000800:00000010:3.0:1788972567.112793:0:102129:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff9a73d0f91200. 00000800:00000010:3.0:1788972567.112893:0:102129:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff9a73d0f91200. 00000800:00000010:3.0:1788972567.112898:0:102129:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at ffff9a73c96019c0. 00000800:00000010:3.0:1788972567.112927:0:102129:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at ffff9a73c7ced760. 00000400:00000200:1.0:1788972567.149876:0:102136:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788972567.149940:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.149950:0:102133:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.47@tcp : GET - for me 00000400:00000200:1.0:1788972567.149962:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.149968:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000010:1.0:1788972567.149976:0:102133:0:(peer.c:230:lnet_peer_alloc()) alloc '(lp)': 360 at ffff9a73d8002800. 00000400:00000200:1.0:1788972567.149997:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.150000:0:102133:0:(peer.c:266:lnet_peer_alloc()) ffff9a73d8002800 nid 192.168.203.47@tcp 00000400:00000010:1.0:1788972567.150004:0:102133:0:(peer.c:194:lnet_peer_net_alloc()) alloc '(lpn)': 72 at ffff9a73c4106060. 00000400:00000200:1.0:1788972567.150007:0:102133:0:(peer.c:203:lnet_peer_net_alloc()) ffff9a73c4106060 net tcp 00000400:00000200:1.0:1788972567.150010:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000010:1.0:1788972567.150014:0:102133:0:(peer.c:138:lnet_peer_ni_alloc()) alloc '(lpni)': 480 at ffff9a73d8002400. 00000400:00000200:1.0:1788972567.150019:0:102133:0:(peer.c:184:lnet_peer_ni_alloc()) ffff9a73d8002400 nid 192.168.203.47@tcp 00000400:00000200:1.0:1788972567.150023:0:102133:0:(peer.c:1216:lnet_peer_ni_set_non_mr_pref_nid()) peer 192.168.203.47@tcp nid 192.168.203.50@tcp: 0 00000400:00000200:1.0:1788972567.150031:0:102133:0:(peer.c:1849:lnet_peer_attach_peer_ni()) peer 192.168.203.47@tcp NID 192.168.203.47@tcp flags 0x0 00000400:00000200:1.0:1788972567.150035:0:102133:0:(peer.c:2228:lnet_peer_ni_traffic_add()) peer 192.168.203.47@tcp: 0 00000400:00000200:1.0:1788972567.150041:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.150048:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.150052:0:102133:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.47@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788972567.150063:0:102133:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.47@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1788972567.150075:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.150080:0:102133:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.50@tcp to NMR: 192.168.203.47@tcp local destination 00000400:00000200:1.0:1788972567.150087:0:102133:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [842:842:256:4294967295] 192.168.203.47@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:1.0:1788972567.150091:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.150100:0:102133:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:192.168.203.50@tcp) -> 192.168.203.47@tcp(192.168.203.47@tcp:192.168.203.47@tcp) 192.168.203.47@tcp : REPLY try# 0 00000800:00000200:1.0:1788972567.150107:0:102133:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.203.47@tcp 00000800:00000010:1.0:1788972567.150113:0:102133:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c500. 00000800:00000200:1.0:1788972567.150120:0:102133:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c9601840] -> 12345-192.168.203.47@tcp (2) 00000800:00000200:1.0:1788972567.150127:0:102133:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.47@tcp ip 192.168.203.47:1023 00000800:00000200:1.0:1788972567.150132:0:102133:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c500 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:1.0:1788972567.150231:0:102133:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788972567.150235:0:102133:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c500. 00000400:00000200:1.0:1788972567.150242:0:102133:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972567.150248:0:102133:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: REPLY: OK 00000400:00000200:1.0:1788972567.156155:0:102136:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.50@tcp from 254.3.0.2 00000800:00000010:1.0:1788972567.156179:0:102136:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at ffff9a73c70682e0. 00000800:00000010:3.0:1788972567.156335:0:102130:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at ffff9a73c5dca000. 00000800:00000010:3.0:1788972567.156342:0:102130:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff9a73c756b240. 00000400:00000200:3.0:1788972567.157988:0:102130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:3.0:1788972567.157994:0:102130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000800:00000010:3.0:1788972567.158002:0:102130:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff9a73c756b9c0. 00000800:00000200:3.1F:1788972567.158015:0:102130:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c9601840] -> 12345-192.168.203.47@tcp (2) 00000800:00000200:3.1:1788972567.158036:0:102130:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.47@tcp ffff9a73c756b9c0 deleted 00000800:00000010:3.1:1788972567.158038:0:102130:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at ffff9a73c756b9c0. 00000800:00000200:3.0:1788972567.158051:0:102130:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.47@tcp p 3.x 192.168.203.50 -> 192.168.203.47:1022 incarnation:1788972565726100512 sched[0] 00000800:00000010:3.0:1788972567.158059:0:102130:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff9a73d0f91e40. 00000800:00000010:3.0:1788972567.158120:0:102130:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff9a73d0f91e40. 00000800:00000010:3.0:1788972567.158125:0:102130:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at ffff9a73c756b240. 00000800:00000010:3.0:1788972567.158153:0:102130:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at ffff9a73c70682e0. 00000400:00000200:1.0:1788972567.163667:0:102136:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788972567.171611:0:102136:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.50@tcp from 253.3.0.2 00000800:00000010:1.0:1788972567.171622:0:102136:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at ffff9a73c7068f60. 00000800:00000010:0.0:1788972567.171740:0:102128:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at ffff9a73c4c02000. 00000800:00000010:0.0:1788972567.171748:0:102128:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff9a73c47e3840. 00000400:00000200:0.0:1788972567.174511:0:102128:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.174519:0:102128:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000800:00000010:0.0:1788972567.174530:0:102128:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff9a73c47e3780. 00000800:00000200:0.1F:1788972567.174542:0:102128:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c9601840] -> 12345-192.168.203.47@tcp (3) 00000800:00000200:0.1:1788972567.174557:0:102128:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.47@tcp ffff9a73c47e3780 deleted 00000800:00000010:0.1:1788972567.174559:0:102128:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at ffff9a73c47e3780. 00000800:00000200:0.0:1788972567.174572:0:102128:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.47@tcp p 3.x 192.168.203.50 -> 192.168.203.47:1021 incarnation:1788972565726100512 sched[0] 00000800:00000010:0.0:1788972567.174581:0:102128:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff9a73c6508ac0. 00000800:00000010:0.0:1788972567.174671:0:102128:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff9a73c6508ac0. 00000800:00000010:0.0:1788972567.174676:0:102128:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at ffff9a73c47e3840. 00000800:00000010:0.0:1788972567.174707:0:102128:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at ffff9a73c7068f60. 00000400:00000200:0.0:1788972567.213115:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.213128:0:102133:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.47@tcp : PUT - for me 00000400:00000200:0.0:1788972567.213140:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.213145:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.213151:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.213156:0:102133:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.47@tcp of length 48 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1788972567.213179:0:102133:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.203.47@tcp of length 48/48 into md 0x19 [1] + 0 00000400:00000010:0.0:1788972567.214526:0:102133:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c014b370. 00000400:00000200:0.0:1788972567.214543:0:102133:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md ffff9a73d13a8990 00000400:00000200:0.0:1788972567.214575:0:102133:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788972567.214585:0:102133:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: PUT: OK 00000400:00000200:0.0:1788972567.214593:0:102133:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8990 00000400:00000010:0.0:1788972567.214596:0:102133:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8990. 00000400:00000200:0.0:1788972567.214602:0:102133:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:0.0:1788972567.214608:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.214623:0:102133:0:(peer.c:2835:lnet_peer_push_event()) peer 192.168.203.47@tcp(ffff9a73d8002800) is MR capable 00000400:00000200:0.0:1788972567.214628:0:102133:0:(peer.c:1241:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.203.47@tcp: 0 00000400:00000010:0.0:1788972567.214644:0:102133:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at ffff9a73c65089c0. 00000400:00000200:0.0:1788972567.214648:0:102133:0:(peer.c:2897:lnet_peer_push_event()) Received Push 192.168.203.47@tcp 5 00000400:00000200:0.0:1788972567.214668:0:102133:0:(peer.c:2658:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.47@tcp: 0 00000400:00000200:0.0:1788972567.214680:0:102133:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1788972567.214684:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.214690:0:102133:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.50@tcp to MR: 192.168.203.47@tcp local destination 00000400:00000200:0.0:1788972567.214696:0:102133:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [843:843:256:4294967295] 192.168.203.47@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:0.0:1788972567.214700:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.214710:0:102133:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:192.168.203.50@tcp) -> 192.168.203.47@tcp(192.168.203.47@tcp:192.168.203.47@tcp) 192.168.203.47@tcp : ACK try# 0 00000800:00000200:0.0:1788972567.214717:0:102133:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.203.47@tcp 00000800:00000010:0.0:1788972567.214724:0:102133:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7600. 00000800:00000200:0.0:1788972567.214731:0:102133:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c9601840] -> 12345-192.168.203.47@tcp (4) 00000800:00000200:0.0:1788972567.214740:0:102133:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.47@tcp ip 192.168.203.47:1023 00000800:00000200:0.0:1788972567.214745:0:102133:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7600 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1788972567.214878:0:102133:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1788972567.214883:0:102133:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7600. 00000400:00000200:0.0:1788972567.214886:0:102133:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788972567.214890:0:102133:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: ACK: OK 00000400:00000200:1.0:1788972567.217998:0:102138:0:(peer.c:4285:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:1.0:1788972567.218020:0:102138:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d2c0. 00000400:00000010:1.0:1788972567.218030:0:102138:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000400:00000200:1.0:1788972567.218040:0:102138:0:(api-ni.c:2383:lnet_push_target_post()) posted push target ffff9a73c137cc00 00000400:00000200:1.0:1788972567.218050:0:102138:0:(peer.c:4397:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0xc1 00000400:00000010:1.0:1788972567.218060:0:102138:0:(peer.c:3519:lnet_peer_merge_data()) alloc '(curnis)': 40 at ffff9a73c82e2d80. 00000400:00000010:1.0:1788972567.218064:0:102138:0:(peer.c:3520:lnet_peer_merge_data()) alloc '(addnis)': 48 at ffff9a73c82e26c0. 00000400:00000010:1.0:1788972567.218066:0:102138:0:(peer.c:3521:lnet_peer_merge_data()) alloc '(delnis)': 40 at ffff9a73c82e2f80. 00000400:00000200:1.0:1788972567.218073:0:102138:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.218079:0:102138:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000010:1.0:1788972567.218081:0:102138:0:(peer.c:3668:lnet_peer_merge_data()) kfreed 'curnis': 40 at ffff9a73c82e2d80. 00000400:00000010:1.0:1788972567.218084:0:102138:0:(peer.c:3669:lnet_peer_merge_data()) kfreed 'addnis': 48 at ffff9a73c82e26c0. 00000400:00000010:1.0:1788972567.218086:0:102138:0:(peer.c:3670:lnet_peer_merge_data()) kfreed 'delnis': 40 at ffff9a73c82e2f80. 00000400:00000010:1.0:1788972567.218089:0:102138:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at ffff9a73c65089c0. 00000400:00000200:1.0:1788972567.218093:0:102138:0:(peer.c:3673:lnet_peer_merge_data()) peer 192.168.203.47@tcp (ffff9a73d8002800): 0 00000400:00000200:1.0:1788972567.218096:0:102138:0:(peer.c:3985:lnet_peer_data_present()) peer 192.168.203.47@tcp(ffff9a73d8002800): 0. state = 0x141 00000400:00000200:1.0:1788972567.218099:0:102138:0:(peer.c:4415:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x141 rc 1 00000400:00000200:1.0:1788972567.218102:0:102138:0:(peer.c:4397:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x141 00000400:00000010:1.0:1788972567.218107:0:102138:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8. 00000400:00000010:1.0:1788972567.218116:0:102138:0:(lib-lnet.h:449:lnet_rspt_alloc()) alloc 'rspt': 64 at ffff9a73c381b000. 00000400:00000200:1.0:1788972567.218120:0:102138:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.47@tcp 00000400:00000200:1.0:1788972567.218130:0:102138:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.218136:0:102138:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.47@tcp local destination 00000400:00000200:1.0:1788972567.218142:0:102138:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972567.218147:0:102138:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972567.218151:0:102138:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972567.218154:0:102138:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.47@tcp 00000400:00000200:1.0:1788972567.218158:0:102138:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [844:844:256:4294967295] 192.168.203.47@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:1.0:1788972567.218162:0:102138:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972567.218167:0:102138:0:(lib-move.c:1846:lnet_handle_send()) rspt_next_hop_nid = 192.168.203.47@tcp 00000400:00000200:1.0:1788972567.218175:0:102138:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.47@tcp(192.168.203.47@tcp:192.168.203.47@tcp) : PUT try# 0 00000800:00000200:1.0:1788972567.218197:0:102138:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.203.47@tcp 00000800:00000010:1.0:1788972567.218203:0:102138:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c500. 00000800:00000200:1.0:1788972567.218211:0:102138:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c9601840] -> 12345-192.168.203.47@tcp (4) 00000800:00000200:1.0:1788972567.218219:0:102138:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.47@tcp ip 192.168.203.47:1023 00000800:00000200:1.0:1788972567.218225:0:102138:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c500 type 1, nob 144 niov 1 nkiov 1 00000400:00000200:1.0:1788972567.218238:0:102138:0:(peer.c:4219:lnet_peer_send_push()) peer 192.168.203.47@tcp 00000400:00000200:1.0:1788972567.218241:0:102138:0:(peer.c:4415:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x541 rc 0 00000800:00000200:1.0:1788972567.218937:0:102132:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788972567.218943:0:102132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c500. 00000400:00000200:1.0:1788972567.218952:0:102132:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972567.218959:0:102132:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: PUT: OK 00000400:00000200:1.0:1788972567.218965:0:102132:0:(peer.c:3334:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1788972567.218968:0:102132:0:(peer.c:3286:lnet_discovery_event_send()) Push Send to 192.168.203.47@tcp: 0 00000400:00000200:0.0:1788972567.229330:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.229367:0:102133:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.47@tcp : ACK - for me 00000400:00000200:0.0:1788972567.229379:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.229385:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:0.0:1788972567.229397:0:102133:0:(lib-move.c:4539:lnet_parse_ack()) 192.168.203.50@tcp: ACK from 12345-192.168.203.47@tcp into md 0xf21 00000400:00000200:0.0:1788972567.229407:0:102133:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1788972567.229413:0:102133:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: ACK: OK 00000400:00000200:0.0:1788972567.229424:0:102133:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8 00000400:00000010:0.0:1788972567.229428:0:102133:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8. 00000400:00000200:0.0:1788972567.229433:0:102133:0:(peer.c:3334:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:0.0:1788972567.229436:0:102133:0:(peer.c:3052:lnet_discovery_event_ack()) peer 192.168.203.47@tcp ev->status 0 00000400:00000200:0.0:1788972567.229442:0:102133:0:(peer.c:2658:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.47@tcp: -114 00000400:00000200:1.0:1788972567.229492:0:102138:0:(peer.c:4285:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1788972567.229500:0:102138:0:(peer.c:4397:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x141 00000400:00000200:1.0:1788972567.229504:0:102138:0:(peer.c:4137:lnet_peer_discovered()) peer 192.168.203.47@tcp 00000400:00000200:1.0:1788972567.229507:0:102138:0:(peer.c:4415:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x111 rc 0 00000400:00000200:1.0:1788972567.229511:0:102138:0:(peer.c:2674:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.203.47@tcp 00000100:00000010:2.0:1788972567.826110:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106f00. 00000100:00000010:3.0:1788972567.841107:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c41065a0. 00000001:02000400:1.0:1788972568.032896:0:131429:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 609: lnet_legacy2yaml: error on no input ========================================================== 12:49:26 (1788972566) 00000100:00000010:3.0:1788972571.923276:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788972571.923294:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 00000100:00000010:3.0:1788972571.923324:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788972571.923329:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1be00. 00000100:00000010:3.0:1788972571.923335:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788972571.923339:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b000. 02000000:00000010:3.0:1788972571.923366:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c33a1e00. 00000400:00000200:3.0:1788972571.923375:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972571.923379:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972571.923387:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788972571.923394:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620992, portal 10 00000400:00000010:3.0:1788972571.923402:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873088. 00000100:00000200:3.0:1788972571.923414:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189620992, offset 0 00000400:00000200:3.0:1788972571.923424:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972571.923433:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.923440:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972571.923445:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972571.923449:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972571.923453:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972571.923456:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972571.923459:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [845:845:256:4294967295] 192.168.203.150@tcp LPNI seq info [842:842:8:4294967295] 00000400:00000200:3.0:1788972571.923463:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.923472:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972571.923477:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972571.923482:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bb00. 00000800:00000200:3.0:1788972571.923489:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972571.923496:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972571.923501:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972571.923513:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972571.923518:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654dd20. 00000100:00000010:3.0:1788972571.923527:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d780. 02000000:00000010:3.0:1788972571.923547:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c33a0000. 00000400:00000200:3.0:1788972571.923551:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972571.923554:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972571.923558:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972571.923561:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189620864, portal 25 00000400:00000010:3.0:1788972571.923564:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48732a8. 00000100:00000200:3.0:1788972571.923566:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189620864, offset 0 00000400:00000200:3.0:1788972571.923571:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972571.923575:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.923578:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972571.923581:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972571.923584:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972571.923592:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972571.923595:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972571.923606:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [846:846:255:4294967295] 192.168.203.150@tcp LPNI seq info [843:843:7:4294967295] 00000400:00000200:3.0:1788972571.923611:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.923617:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972571.923622:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972571.923626:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bd00. 00000800:00000200:3.0:1788972571.923629:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972571.923634:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972571.923636:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972571.923640:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972571.923653:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003400. 00000400:00000200:3.0:1788972571.923657:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972571.923659:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788972571.923662:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788972571.923665:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621120, portal 10 00000400:00000010:3.0:1788972571.923670:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972571.923672:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189621120, offset 0 00000400:00000200:3.0:1788972571.923676:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972571.923680:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.923683:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972571.923685:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972571.923688:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972571.923690:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972571.923692:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972571.923695:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [847:847:254:4294967295] 192.168.203.150@tcp LPNI seq info [844:844:6:4294967295] 00000400:00000200:3.0:1788972571.923699:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.923704:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972571.923708:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972571.923711:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b900. 00000800:00000200:3.0:1788972571.923722:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972571.923726:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972571.923728:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972571.923734:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972571.924192:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972571.924197:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bb00. 00000400:00000200:2.0:1788972571.924202:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972571.924207:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972571.924214:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873088 00000400:00000010:2.0:1788972571.924216:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873088. 00000100:00000200:2.0:1788972571.924226:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189620992/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972571.924249:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972571.924251:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bd00. 00000400:00000200:2.0:1788972571.924254:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972571.924257:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972571.924259:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48732a8 00000400:00000010:2.0:1788972571.924261:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48732a8. 00000100:00000200:2.0:1788972571.924266:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790e00 x1875872189620864/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972571.924335:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972571.924338:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b900. 00000400:00000200:2.0:1788972571.924341:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972571.924345:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972571.924348:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:2.0:1788972571.924350:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000100:00000200:2.0:1788972571.924354:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790a80 x1875872189621120/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972571.947383:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947392:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972571.947401:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947406:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947417:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947421:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781a280 00000400:00000200:3.0:1788972571.947427:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947434:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2985 [1] + 224 00000400:00000200:3.0:1788972571.947445:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972571.947449:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972571.947462:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790e00 x1875872189620864/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972571.947490:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947494:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972571.947498:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947500:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947502:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947505:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a300 00000400:00000200:3.0:1788972571.947508:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947512:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2975 [1] + 224 00000400:00000200:3.0:1788972571.947519:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972571.947522:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972571.947528:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189620992/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972571.947541:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947557:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972571.947561:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947564:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947567:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947569:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a380 00000400:00000200:3.0:1788972571.947573:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972571.947576:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2995 [1] + 224 00000400:00000200:3.0:1788972571.947584:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972571.947588:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972571.947594:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790a80 x1875872189621120/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972571.947638:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972571.947642:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000010:3.0:1788972571.947644:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972571.947650:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790e00 x1875872189620864/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972571.947658:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790e00 x1875872189620864/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972571.947682:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:3.0:1788972571.947685:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0 00000400:00000010:3.0:1788972571.947686:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788972571.947690:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790a80 x1875872189621120/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972571.947699:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790a80 x1875872189621120/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972571.947713:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972571.947715:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873198 00000400:00000010:3.0:1788972571.947717:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788972571.947721:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189620992/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972571.947741:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189620992/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972587 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972571.947760:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c33a0000. 02000000:00000010:3.0:1788972571.947766:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:3.0:1788972571.947770:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788972571.947774:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003400. 02000000:00000010:3.0:1788972571.947777:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b000. 00000100:00000010:3.0:1788972571.947779:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788972571.947781:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c33a1e00. 02000000:00000010:3.0:1788972571.947784:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1be00. 00000100:00000010:3.0:1788972571.947786:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 00000100:00000010:3.0:1788972572.945888:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d780. 00000100:00000010:2.0:1788972572.946226:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654dd20. 00000001:02000400:0.0:1788972576.868900:0:132009:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 610: lnet_legacy2yaml: multiple files and inline, check order ========================================================== 12:49:35 (1788972575) 00000100:00000010:2.0:1788972577.042529:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972577.042549:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387700. 00000100:00000010:2.0:1788972577.042589:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972577.042594:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387100. 00000100:00000010:2.0:1788972577.042600:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972577.042605:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387800. 02000000:00000010:2.0:1788972577.042704:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef200. 00000400:00000200:2.0:1788972577.042713:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972577.042718:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972577.042725:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972577.042732:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621248, portal 25 00000400:00000010:2.0:1788972577.042738:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972577.042741:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189621248, offset 0 00000400:00000200:2.0:1788972577.042750:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972577.042759:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.042767:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972577.042777:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972577.042781:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972577.042785:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972577.042788:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972577.042792:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [848:848:256:4294967295] 192.168.203.150@tcp LPNI seq info [845:845:8:4294967295] 00000400:00000200:2.0:1788972577.042798:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.042807:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972577.042840:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972577.042847:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387d00. 00000800:00000200:2.0:1788972577.042855:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972577.042862:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972577.042867:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972577.042891:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972577.042897:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78540. 00000100:00000010:2.0:1788972577.042907:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78900. 02000000:00000010:2.0:1788972577.042921:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cefa00. 00000400:00000200:2.0:1788972577.042925:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972577.042929:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580. 00000400:00000010:2.0:1788972577.042934:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972577.042937:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621504, portal 10 00000400:00000010:2.0:1788972577.042942:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972577.042944:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189621504, offset 0 00000400:00000200:2.0:1788972577.042950:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972577.042954:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.042958:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972577.042961:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972577.042964:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972577.042966:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972577.042969:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972577.042972:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [849:849:255:4294967295] 192.168.203.150@tcp LPNI seq info [846:846:7:4294967295] 00000400:00000200:2.0:1788972577.042976:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.042982:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972577.042986:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972577.042992:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387b00. 00000800:00000200:2.0:1788972577.042996:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972577.043000:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972577.043003:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972577.043008:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972577.043043:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef400. 00000400:00000200:2.0:1788972577.043047:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972577.043050:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000010:2.0:1788972577.043055:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972577.043058:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621376, portal 10 00000400:00000010:2.0:1788972577.043070:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788972577.043073:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189621376, offset 0 00000400:00000200:2.0:1788972577.043079:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972577.043084:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.043087:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972577.043091:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972577.043094:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972577.043097:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972577.043099:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972577.043102:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [850:850:254:4294967295] 192.168.203.150@tcp LPNI seq info [847:847:6:4294967295] 00000400:00000200:2.0:1788972577.043106:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.043113:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972577.043117:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972577.043120:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387500. 00000800:00000200:2.0:1788972577.043124:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972577.043129:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972577.043149:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972577.043155:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972577.043202:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972577.043206:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387d00. 00000400:00000200:2.0:1788972577.043211:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972577.043216:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972577.043221:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788972577.043223:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972577.043232:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189621248/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972577.043255:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972577.043258:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387b00. 00000400:00000200:2.0:1788972577.043261:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972577.043264:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972577.043267:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972577.043275:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972577.043282:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189621504/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972577.043347:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972577.043350:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387500. 00000400:00000200:2.0:1788972577.043353:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972577.043356:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972577.043359:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:2.0:1788972577.043361:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788972577.043366:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189621376/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972577.047625:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.047635:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972577.047644:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.047648:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.047653:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.047657:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781a400 00000400:00000200:2.0:1788972577.047663:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.047668:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x29a5 [1] + 224 00000400:00000200:2.0:1788972577.047683:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972577.047687:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972577.047701:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189621248/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972577.047742:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:2.0:1788972577.047747:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788972577.047749:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972577.047754:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189621248/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972577.047763:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189621248/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972577.047803:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef200. 02000000:00000010:2.0:1788972577.047809:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387700. 00000100:00000010:2.0:1788972577.047838:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000400:00000200:2.0:1788972577.053924:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.053935:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972577.053945:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.053950:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.053956:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.053960:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a500 00000400:00000200:2.0:1788972577.053966:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.053973:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x29b5 [1] + 224 00000400:00000200:2.0:1788972577.053985:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972577.053989:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972577.054003:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189621504/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972577.054031:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.054035:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972577.054039:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.054041:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.054044:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.054046:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a480 00000400:00000200:2.0:1788972577.054049:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972577.054053:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x29c5 [1] + 224 00000400:00000200:2.0:1788972577.054061:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972577.054064:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972577.054070:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189621376/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972577.054099:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:2.0:1788972577.054111:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972577.054114:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972577.054120:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189621376/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972577.054130:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189621376/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972577.054174:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580. 00000400:00000200:2.0:1788972577.054178:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972577.054180:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972577.054185:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189621504/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972577.054193:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189621504/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972593 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972577.054208:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef400. 02000000:00000010:2.0:1788972577.054214:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387100. 00000100:00000010:2.0:1788972577.054219:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972577.054223:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cefa00. 02000000:00000010:2.0:1788972577.054225:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387800. 00000100:00000010:2.0:1788972577.054228:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 00000100:00000010:2.0:1788972578.066958:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78540. 00000100:00000010:3.0:1788972578.072174:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78900. 00000800:00000200:1.0:1788972578.868043:0:102132:0:(socklnd_cb.c:1106:ksocknal_process_receive()) [ffff9a73c4c02000] EOF from 12345-192.168.203.47@tcp ip 192.168.203.47:1021 00000800:00000200:3.0:1788972578.868598:0:102131:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff9a73c4c02000 00000800:00000010:3.0:1788972578.868602:0:102131:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff9a73c4c02000. 00000800:00000200:3.0:1788972578.868607:0:102131:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff9a73c5dca000 00000800:00000010:3.0:1788972578.868608:0:102131:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff9a73c5dca000. 00000800:00000200:3.0:1788972578.868611:0:102131:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff9a73c5dce000 00000800:00000200:3.0:1788972578.868615:0:102131:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.47@tcp ffff9a73c9601840 deleted 00000800:00000010:3.0:1788972578.868618:0:102131:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at ffff9a73c9601840. 00000800:00000010:3.0:1788972578.868621:0:102131:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff9a73c5dce000. 00000400:00000200:1.0:1788972580.134051:0:102136:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:1.0:1788972580.134088:0:102136:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.50@tcp from 255.3.0.2 00000800:00000010:1.0:1788972580.134095:0:102136:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at ffff9a73c7068860. 00000800:00000010:3.0:1788972580.134183:0:102127:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at ffff9a73c5dce000. 00000800:00000010:3.0:1788972580.134188:0:102127:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff9a73c486f000. 00000400:00000200:2.0:1788972580.164729:0:102127:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:2.0:1788972580.164735:0:102127:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000800:00000010:2.0:1788972580.164742:0:102127:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff9a73c47e3a80. 00000800:00000200:2.0:1788972580.164756:0:102127:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.47@tcp p 3.x 192.168.203.50 -> 192.168.203.47:1023 incarnation:1788972578931816568 sched[0] 00000800:00000010:2.0:1788972580.164764:0:102127:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff9a73c4ca9740. 00000800:00000010:2.0:1788972580.164849:0:102127:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff9a73c4ca9740. 00000800:00000010:2.0:1788972580.164853:0:102127:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at ffff9a73c486f000. 00000800:00000010:2.0:1788972580.164880:0:102127:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at ffff9a73c7068860. 00000400:00000200:1.0:1788972580.186769:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.186779:0:102133:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.47@tcp : GET - for me 00000400:00000200:1.0:1788972580.186790:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.186794:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.186799:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.186803:0:102133:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.47@tcp of length 272 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788972580.186860:0:102133:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming get index 0 from 12345-192.168.203.47@tcp of length 48/272 into md 0x1 [1] + 0 00000400:00000200:1.0:1788972580.186871:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.186889:0:102133:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.50@tcp to MR: 192.168.203.47@tcp local destination 00000400:00000200:1.0:1788972580.186895:0:102133:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [851:851:256:4294967295] 192.168.203.47@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:1.0:1788972580.186899:0:102133:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.186907:0:102133:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:192.168.203.50@tcp) -> 192.168.203.47@tcp(192.168.203.47@tcp:192.168.203.47@tcp) 192.168.203.47@tcp : REPLY try# 0 00000800:00000200:1.0:1788972580.186913:0:102133:0:(socklnd_cb.c:931:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.203.47@tcp 00000800:00000010:1.0:1788972580.186919:0:102133:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff300. 00000800:00000200:1.0:1788972580.186925:0:102133:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c47e3a80] -> 12345-192.168.203.47@tcp (2) 00000800:00000200:1.0:1788972580.186931:0:102133:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.47@tcp ip 192.168.203.47:1023 00000800:00000200:1.0:1788972580.186936:0:102133:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff300 type 3, nob 144 niov 1 nkiov 1 00000800:00000200:1.0:1788972580.186993:0:102133:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788972580.186997:0:102133:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff300. 00000400:00000200:1.0:1788972580.187001:0:102133:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972580.187007:0:102133:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: REPLY: OK 00000400:00000200:2.0:1788972580.214142:0:102136:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:2.0:1788972580.214177:0:102136:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.50@tcp from 254.3.0.2 00000800:00000010:2.0:1788972580.214187:0:102136:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at ffff9a73c7ced880. 00000800:00000010:0.0:1788972580.215021:0:102129:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at ffff9a73c4c04000. 00000800:00000010:0.0:1788972580.215028:0:102129:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff9a73c756b000. 00000400:00000200:1.0:1788972580.220703:0:102129:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.220708:0:102129:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000800:00000010:1.0:1788972580.220715:0:102129:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff9a73d80eaf00. 00000800:00000200:1.1F:1788972580.220724:0:102129:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c47e3a80] -> 12345-192.168.203.47@tcp (2) 00000800:00000200:1.1:1788972580.220733:0:102129:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.47@tcp ffff9a73d80eaf00 deleted 00000800:00000010:1.1:1788972580.220735:0:102129:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at ffff9a73d80eaf00. 00000800:00000200:1.0:1788972580.220745:0:102129:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.47@tcp p 3.x 192.168.203.50 -> 192.168.203.47:1022 incarnation:1788972578931816568 sched[0] 00000800:00000010:1.0:1788972580.220751:0:102129:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff9a73c82e2900. 00000800:00000010:1.0:1788972580.220804:0:102129:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff9a73c82e2900. 00000800:00000010:1.0:1788972580.220808:0:102129:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at ffff9a73c756b000. 00000800:00000010:1.0:1788972580.220856:0:102129:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at ffff9a73c7ced880. 00000400:00000200:1.0:1788972580.223474:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223483:0:102132:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.47@tcp : PUT - for me 00000400:00000200:1.0:1788972580.223491:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223495:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223500:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223505:0:102132:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.47@tcp of length 48 into portal 0 MB=0x8000000000000000 00000400:00000200:1.0:1788972580.223514:0:102132:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.203.47@tcp of length 48/48 into md 0xf19 [1] + 0 00000400:00000010:1.0:1788972580.223519:0:102132:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d2c0. 00000400:00000200:1.0:1788972580.223530:0:102132:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md ffff9a73c7525198 00000400:00000200:1.0:1788972580.223552:0:102132:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972580.223558:0:102132:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: PUT: OK 00000400:00000200:1.0:1788972580.223563:0:102132:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:1.0:1788972580.223566:0:102132:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000400:00000200:1.0:1788972580.223569:0:102132:0:(api-ni.c:2393:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:1.0:1788972580.223574:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223578:0:102132:0:(peer.c:2823:lnet_peer_push_event()) peer 192.168.203.47@tcp(ffff9a73d8002800) is MR 00000400:00000010:1.0:1788972580.223583:0:102132:0:(api-ni.c:1874:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at ffff9a73c82e2900. 00000400:00000200:1.0:1788972580.223586:0:102132:0:(peer.c:2897:lnet_peer_push_event()) Received Push 192.168.203.47@tcp 7 00000400:00000200:1.0:1788972580.223597:0:102132:0:(peer.c:2658:lnet_peer_queue_for_discovery()) Queue peer 192.168.203.47@tcp: 0 00000400:00000200:1.0:1788972580.223604:0:102132:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:1.0:1788972580.223608:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223612:0:102132:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source Specified: 192.168.203.50@tcp to MR: 192.168.203.47@tcp local destination 00000400:00000200:1.0:1788972580.223618:0:102132:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [852:852:256:4294967295] 192.168.203.47@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:1.0:1788972580.223622:0:102132:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223629:0:102132:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:192.168.203.50@tcp) -> 192.168.203.47@tcp(192.168.203.47@tcp:192.168.203.47@tcp) 192.168.203.47@tcp : ACK try# 0 00000800:00000200:1.0:1788972580.223636:0:102132:0:(socklnd_cb.c:931:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.203.47@tcp 00000800:00000010:1.0:1788972580.223640:0:102132:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff300. 00000800:00000200:1.0:1788972580.223645:0:102132:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c47e3a80] -> 12345-192.168.203.47@tcp (3) 00000800:00000200:1.0:1788972580.223651:0:102132:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.47@tcp ip 192.168.203.47:1023 00000800:00000200:1.0:1788972580.223656:0:102132:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff300 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:1.0:1788972580.223727:0:102132:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1788972580.223732:0:102132:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff300. 00000400:00000200:1.0:1788972580.223735:0:102132:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972580.223739:0:102132:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.47@tcp: ACK: OK 00000400:00000200:1.0:1788972580.223756:0:102138:0:(peer.c:4285:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:1.0:1788972580.223762:0:102138:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d2c0. 00000400:00000010:1.0:1788972580.223766:0:102138:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000400:00000200:1.0:1788972580.223771:0:102138:0:(api-ni.c:2383:lnet_push_target_post()) posted push target ffff9a73c137cc00 00000400:00000200:1.0:1788972580.223775:0:102138:0:(peer.c:4397:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0xd1 00000400:00000010:1.0:1788972580.223787:0:102138:0:(peer.c:3519:lnet_peer_merge_data()) alloc '(curnis)': 40 at ffff9a73c82e2180. 00000400:00000010:1.0:1788972580.223790:0:102138:0:(peer.c:3520:lnet_peer_merge_data()) alloc '(addnis)': 48 at ffff9a73c82e25c0. 00000400:00000010:1.0:1788972580.223792:0:102138:0:(peer.c:3521:lnet_peer_merge_data()) alloc '(delnis)': 40 at ffff9a73c82e2200. 00000400:00000200:1.0:1788972580.223795:0:102138:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.223799:0:102138:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000010:1.0:1788972580.223801:0:102138:0:(peer.c:3668:lnet_peer_merge_data()) kfreed 'curnis': 40 at ffff9a73c82e2180. 00000400:00000010:1.0:1788972580.223803:0:102138:0:(peer.c:3669:lnet_peer_merge_data()) kfreed 'addnis': 48 at ffff9a73c82e25c0. 00000400:00000010:1.0:1788972580.223805:0:102138:0:(peer.c:3670:lnet_peer_merge_data()) kfreed 'delnis': 40 at ffff9a73c82e2200. 00000400:00000010:1.0:1788972580.223807:0:102138:0:(api-ni.c:1892:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at ffff9a73c82e2900. 00000400:00000200:1.0:1788972580.223810:0:102138:0:(peer.c:3673:lnet_peer_merge_data()) peer 192.168.203.47@tcp (ffff9a73d8002800): 0 00000400:00000200:1.0:1788972580.223836:0:102138:0:(peer.c:3985:lnet_peer_data_present()) peer 192.168.203.47@tcp(ffff9a73d8002800): 0. state = 0x151 00000400:00000200:1.0:1788972580.223840:0:102138:0:(peer.c:4415:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x151 rc 1 00000400:00000200:1.0:1788972580.223843:0:102138:0:(peer.c:4397:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x151 00000400:00000200:1.0:1788972580.223846:0:102138:0:(peer.c:4137:lnet_peer_discovered()) peer 192.168.203.47@tcp 00000400:00000200:1.0:1788972580.223848:0:102138:0:(peer.c:4415:lnet_peer_discovery()) peer 192.168.203.47@tcp(ffff9a73d8002800) state 0x111 rc 0 00000400:00000200:1.0:1788972580.223851:0:102138:0:(peer.c:2674:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.203.47@tcp 00000400:00000200:2.0:1788972580.234995:0:102136:0:(acceptor.c:578:lnet_acceptor()) Accepted connection on ens2:988 00000400:00000200:2.0:1788972580.242507:0:102136:0:(acceptor.c:502:lnet_accept()) Accept 192.168.203.50@tcp from 253.3.0.2 00000800:00000010:2.0:1788972580.242515:0:102136:0:(socklnd.c:737:ksocknal_accept()) alloc '(cr)': 32 at ffff9a73c7ced000. 00000800:00000010:2.0:1788972580.242953:0:102130:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4728 at ffff9a73c49e4000. 00000800:00000010:2.0:1788972580.242958:0:102130:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff9a73c47e33c0. 00000400:00000200:1.0:1788972580.252520:0:102130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000400:00000200:1.0:1788972580.252526:0:102130:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.47@tcp to cpt 0 00000800:00000010:1.0:1788972580.252534:0:102130:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff9a73d80eaf00. 00000800:00000200:1.1:1788972580.252553:0:102130:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73c47e3a80] -> 12345-192.168.203.47@tcp (3) 00000800:00000200:1.1:1788972580.252557:0:102130:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.47@tcp ffff9a73d80eaf00 deleted 00000800:00000010:1.1:1788972580.252560:0:102130:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at ffff9a73d80eaf00. 00000800:00000200:1.0:1788972580.252570:0:102130:0:(socklnd.c:1182:ksocknal_create_conn()) New conn 12345-192.168.203.47@tcp p 3.x 192.168.203.50 -> 192.168.203.47:1021 incarnation:1788972578931816568 sched[0] 00000800:00000010:1.0:1788972580.252578:0:102130:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff9a73c82e2900. 00000800:00000010:1.0:1788972580.252656:0:102130:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff9a73c82e2900. 00000800:00000010:1.0:1788972580.252661:0:102130:0:(socklnd.c:1190:ksocknal_create_conn()) kfreed 'hello': 144 at ffff9a73c47e33c0. 00000800:00000010:1.0:1788972580.252685:0:102130:0:(socklnd_cb.c:2179:ksocknal_connd()) kfreed 'cr': 32 at ffff9a73c7ced000. 00000100:00000010:2.0:1788972582.162147:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972582.162173:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387e00. 00000100:00000010:2.0:1788972582.162206:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972582.162210:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387900. 00000100:00000010:2.0:1788972582.162216:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972582.162219:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387400. 02000000:00000010:2.0:1788972582.162255:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cee800. 00000400:00000200:2.0:1788972582.162266:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972582.162271:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580. 00000400:00000010:2.0:1788972582.162278:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972582.162287:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621632, portal 25 00000400:00000010:2.0:1788972582.162293:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972582.162296:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189621632, offset 0 00000400:00000200:2.0:1788972582.162307:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972582.162317:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.162326:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972582.162332:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972582.162336:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972582.162341:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972582.162344:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972582.162348:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [853:853:256:4294967295] 192.168.203.150@tcp LPNI seq info [848:848:8:4294967295] 00000400:00000200:2.0:1788972582.162353:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.162364:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972582.162370:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972582.162376:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387b00. 00000800:00000200:2.0:1788972582.162384:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972582.162401:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972582.162407:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972582.162420:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972582.162425:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78060. 00000100:00000010:2.0:1788972582.162438:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c780c0. 02000000:00000010:2.0:1788972582.162448:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef200. 00000400:00000200:2.0:1788972582.162452:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972582.162454:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000010:2.0:1788972582.162458:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972582.162461:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621888, portal 10 00000400:00000010:2.0:1788972582.162466:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788972582.162469:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189621888, offset 0 00000400:00000200:2.0:1788972582.162474:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972582.162477:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.162480:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972582.162483:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972582.162498:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972582.162500:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972582.162503:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972582.162506:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [854:854:255:4294967295] 192.168.203.150@tcp LPNI seq info [849:849:7:4294967295] 00000400:00000200:2.0:1788972582.162510:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.162516:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972582.162521:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972582.162534:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d4603300. 00000800:00000200:2.0:1788972582.162539:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972582.162543:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972582.162546:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d4603300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972582.162552:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972582.162573:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7ceea00. 00000400:00000200:2.0:1788972582.162577:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972582.162585:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972582.162589:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972582.162592:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189621760, portal 10 00000400:00000010:2.0:1788972582.162597:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972582.162599:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189621760, offset 0 00000400:00000200:2.0:1788972582.162604:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972582.162608:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.162612:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972582.162615:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972582.162617:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972582.162619:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972582.162621:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972582.162624:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [855:855:254:4294967295] 192.168.203.150@tcp LPNI seq info [850:850:6:4294967295] 00000400:00000200:2.0:1788972582.162628:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.162634:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972582.162639:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972582.162643:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d4603d00. 00000800:00000200:2.0:1788972582.162647:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972582.162651:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972582.162654:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d4603d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972582.162659:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972582.163040:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972582.163044:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387b00. 00000400:00000200:2.0:1788972582.163050:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972582.163056:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972582.163063:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972582.163065:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972582.163075:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189621632/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972582.163099:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972582.163101:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d4603300. 00000400:00000200:2.0:1788972582.163108:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972582.163111:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972582.163114:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:2.0:1788972582.163115:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788972582.163120:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189621888/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972582.163194:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972582.163196:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d4603d00. 00000400:00000200:2.0:1788972582.163199:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972582.163202:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972582.163204:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788972582.163206:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972582.163210:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189621760/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972582.175131:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175144:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972582.175156:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175161:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175169:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175173:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a680 00000400:00000200:2.0:1788972582.175180:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175188:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x29e5 [1] + 224 00000400:00000200:2.0:1788972582.175203:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972582.175208:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972582.175227:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189621888/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972582.175268:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175272:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972582.175277:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175279:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175291:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175294:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781a580 00000400:00000200:2.0:1788972582.175298:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.175302:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x29d5 [1] + 224 00000400:00000200:2.0:1788972582.175316:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972582.175320:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972582.175328:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189621632/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972582.175374:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580. 00000400:00000200:2.0:1788972582.175380:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972582.175383:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972582.175390:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189621632/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972582.175399:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189621632/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972582.175434:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:2.0:1788972582.175437:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788972582.175439:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972582.175444:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189621888/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972582.175454:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189621888/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972582.175479:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cee800. 02000000:00000010:2.0:1788972582.175498:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387e00. 00000100:00000010:2.0:1788972582.175505:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972582.175510:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef200. 02000000:00000010:2.0:1788972582.175513:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387400. 00000100:00000010:2.0:1788972582.175515:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000400:00000200:2.0:1788972582.190642:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.190664:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972582.190675:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.190681:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.190686:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.190690:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a600 00000400:00000200:2.0:1788972582.190697:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972582.190704:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x29f5 [1] + 224 00000400:00000200:2.0:1788972582.190734:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972582.190740:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972582.190759:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189621760/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972582.190843:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:2.0:1788972582.190849:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972582.190852:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972582.190858:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189621760/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972582.190868:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189621760/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972582.190912:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7ceea00. 02000000:00000010:2.0:1788972582.190919:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387900. 00000100:00000010:2.0:1788972582.190925:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 00000100:00000010:3.0:1788972583.186407:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c780c0. 00000100:00000010:2.0:1788972583.187598:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78060. 00000001:02000400:3.0:1788972585.411643:0:132591:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 611: lnet_legacy2yaml: invalid hopcount ========================================================== 12:49:43 (1788972583) 00000100:00000010:2.0:1788972587.281948:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972587.281972:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d4603900. 00000100:00000010:2.0:1788972587.282015:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972587.282021:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387d00. 00000100:00000010:2.0:1788972587.282028:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972587.282032:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387900. 02000000:00000010:2.0:1788972587.282075:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f800. 00000400:00000200:2.0:1788972587.282086:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972587.282097:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972587.282104:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972587.282112:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622144, portal 10 00000400:00000010:2.0:1788972587.282119:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972587.282121:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189622144, offset 0 00000400:00000200:2.0:1788972587.282130:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972587.282140:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.282148:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972587.282152:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972587.282157:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972587.282160:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972587.282163:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972587.282167:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [856:856:256:4294967295] 192.168.203.150@tcp LPNI seq info [851:851:8:4294967295] 00000400:00000200:2.0:1788972587.282171:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.282181:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972587.282187:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 02000000:00000010:3.0:1788972587.282347:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600. 00000400:00000200:3.0:1788972587.282352:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972587.282356:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972587.282360:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788972587.282368:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622016, portal 25 00000400:00000010:3.0:1788972587.282372:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788972587.282374:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189622016, offset 0 00000400:00000200:3.0:1788972587.282379:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972587.282384:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972587.282388:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972587.282390:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972587.282393:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972587.282395:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972587.282397:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972587.282400:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [857:857:255:4294967295] 192.168.203.150@tcp LPNI seq info [852:852:7:4294967295] 00000400:00000200:3.0:1788972587.282404:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972587.282410:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972587.282415:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972587.282420:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:3.0:1788972587.282426:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972587.282433:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972587.282438:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972587.282453:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972587.282458:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d540. 00000100:00000010:3.0:1788972587.282468:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d180. 02000000:00000010:3.0:1788972587.282479:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8834800. 00000400:00000200:3.0:1788972587.282482:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972587.282484:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210. 00000400:00000010:3.0:1788972587.282488:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972587.282500:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622272, portal 10 00000400:00000010:3.0:1788972587.282504:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972587.282506:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189622272, offset 0 00000400:00000200:3.0:1788972587.282511:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972587.282514:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972587.282517:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972587.282525:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972587.282528:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972587.282530:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972587.282532:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972587.282535:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [858:858:254:4294967295] 192.168.203.150@tcp LPNI seq info [853:853:6:4294967295] 00000400:00000200:3.0:1788972587.282539:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972587.282545:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972587.282549:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972587.282553:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b000. 00000800:00000200:3.0:1788972587.282557:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972587.282561:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972587.282563:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972587.282569:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:2.0:1788972587.282617:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387400. 00000800:00000200:2.0:1788972587.282622:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972587.282625:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972587.282628:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972587.282633:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972587.282680:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972587.282685:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:2.0:1788972587.282691:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972587.282697:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972587.282703:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0 00000400:00000010:2.0:1788972587.282705:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0. 00000100:00000200:2.0:1788972587.282714:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189622016/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972587.282737:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972587.282739:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b000. 00000400:00000200:2.0:1788972587.282747:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972587.282751:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972587.282754:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:2.0:1788972587.282756:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000100:00000200:2.0:1788972587.282761:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189622272/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972587.282852:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972587.282854:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387400. 00000400:00000200:2.0:1788972587.282858:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972587.282862:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972587.282865:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788972587.282867:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972587.282872:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189622144/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972587.289325:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289335:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972587.289344:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289348:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289353:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289357:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a800 00000400:00000200:2.0:1788972587.289363:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289369:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a25 [1] + 224 00000400:00000200:2.0:1788972587.289380:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972587.289385:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972587.289399:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189622272/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972587.289498:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289502:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972587.289507:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289510:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289521:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289524:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a780 00000400:00000200:2.0:1788972587.289528:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289532:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a05 [1] + 224 00000400:00000200:2.0:1788972587.289539:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972587.289542:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972587.289549:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189622144/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972587.289570:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289573:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972587.289577:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289579:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289581:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289583:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781a700 00000400:00000200:2.0:1788972587.289587:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972587.289590:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a15 [1] + 224 00000400:00000200:2.0:1788972587.289599:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972587.289602:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972587.289608:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189622016/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972587.289634:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:2.0:1788972587.289639:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972587.289641:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972587.289647:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189622144/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972587.289656:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189622144/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972587.289687:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f800. 02000000:00000010:2.0:1788972587.289694:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387d00. 00000100:00000010:2.0:1788972587.289705:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000400:00000010:3.0:1788972587.290475:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210. 00000400:00000200:3.0:1788972587.290480:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000010:3.0:1788972587.290482:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972587.290500:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189622272/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972587.290510:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189622272/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972587.290530:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8834800. 02000000:00000010:3.0:1788972587.290534:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387900. 00000100:00000010:3.0:1788972587.290538:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 00000400:00000010:3.0:1788972587.290544:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972587.290546:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873198 00000400:00000010:3.0:1788972587.290548:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788972587.290553:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189622016/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972587.290560:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189622016/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972603 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972587.290585:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600. 02000000:00000010:3.0:1788972587.290589:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d4603900. 00000100:00000010:3.0:1788972587.290592:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 00000100:00000010:3.0:1788972588.305893:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d180. 00000100:00000010:2.0:1788972588.307172:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d540. 00000800:00000200:1.0:1788972588.346795:0:102133:0:(socklnd_cb.c:1106:ksocknal_process_receive()) [ffff9a73c49e4000] EOF from 12345-192.168.203.47@tcp ip 192.168.203.47:1021 00000800:00000200:0.0:1788972588.347286:0:102131:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff9a73c49e4000 00000800:00000010:0.0:1788972588.347294:0:102131:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff9a73c49e4000. 00000800:00000200:0.0:1788972588.347302:0:102131:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff9a73c4c04000 00000800:00000010:0.0:1788972588.347304:0:102131:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff9a73c4c04000. 00000800:00000200:0.0:1788972588.347313:0:102131:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff9a73c5dce000 00000800:00000200:0.0:1788972588.347323:0:102131:0:(socklnd.c:175:ksocknal_destroy_peer()) peer_ni 12345-192.168.203.47@tcp ffff9a73c47e3a80 deleted 00000800:00000010:0.0:1788972588.347326:0:102131:0:(socklnd.c:184:ksocknal_destroy_peer()) kfreed 'peer_ni': 184 at ffff9a73c47e3a80. 00000800:00000010:0.0:1788972588.347330:0:102131:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff9a73c5dce000. 00000100:00000010:1.0:1788972592.405966:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972592.405998:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff100. 00000100:00000010:1.0:1788972592.406053:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972592.406059:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffc00. 00000100:00000010:1.0:1788972592.406066:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972592.406070:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff400. 02000000:00000010:1.0:1788972592.406124:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 00000400:00000200:1.0:1788972592.406149:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972592.406155:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d630. 00000400:00000010:1.0:1788972592.406163:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972592.406173:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622528, portal 10 00000400:00000010:1.0:1788972592.406182:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525990. 00000100:00000200:1.0:1788972592.406187:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189622528, offset 0 00000400:00000200:1.0:1788972592.406199:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972592.406211:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972592.406221:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972592.406229:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972592.406233:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972592.406239:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972592.406242:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972592.406246:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [859:859:256:4294967295] 192.168.203.150@tcp LPNI seq info [854:854:8:4294967295] 00000400:00000200:1.0:1788972592.406251:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972592.406263:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972592.406270:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972592.406276:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffe00. 00000800:00000200:1.0:1788972592.406284:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972592.406293:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972592.406299:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffe00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972592.406324:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972592.406331:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106600. 00000100:00000010:1.0:1788972592.406345:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106660. 02000000:00000010:1.0:1788972592.406374:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03ddc00. 00000400:00000200:1.0:1788972592.406378:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972592.406381:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1db58. 00000400:00000010:1.0:1788972592.406385:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788972592.406388:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622400, portal 25 00000400:00000010:1.0:1788972592.406392:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972592.406395:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189622400, offset 0 00000400:00000200:1.0:1788972592.406406:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972592.406410:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972592.406414:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972592.406416:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972592.406419:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972592.406421:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972592.406423:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972592.406425:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [860:860:255:4294967295] 192.168.203.150@tcp LPNI seq info [855:855:7:4294967295] 00000400:00000200:1.0:1788972592.406429:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972592.406435:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972592.406439:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972592.406443:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cfff00. 00000800:00000200:1.0:1788972592.406447:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972592.406451:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972592.406464:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cfff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972592.406468:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972592.406483:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd400. 00000400:00000200:1.0:1788972592.406486:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972592.406489:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d268. 00000400:00000010:1.0:1788972592.406493:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972592.406495:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622656, portal 10 00000400:00000010:1.0:1788972592.406500:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525908. 00000100:00000200:1.0:1788972592.406502:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189622656, offset 0 00000400:00000200:1.0:1788972592.406507:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972592.406510:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972592.406513:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972592.406516:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972592.406518:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972592.406520:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972592.406522:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972592.406525:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [861:861:254:4294967295] 192.168.203.150@tcp LPNI seq info [856:856:6:4294967295] 00000400:00000200:1.0:1788972592.406534:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972592.406541:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972592.406546:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972592.406549:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff500. 00000800:00000200:1.0:1788972592.406552:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972592.406557:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972592.406559:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972592.406564:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972592.407042:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972592.407049:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffe00. 00000400:00000200:2.0:1788972592.407054:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972592.407061:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972592.407067:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525990 00000400:00000010:2.0:1788972592.407070:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525990. 00000100:00000200:2.0:1788972592.407081:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189622528/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972592.407107:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972592.407109:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cfff00. 00000400:00000200:2.0:1788972592.407112:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972592.407115:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972592.407118:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:2.0:1788972592.407120:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:2.0:1788972592.407123:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1c00 x1875872189622400/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972608 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972592.407409:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972592.407413:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff500. 00000400:00000200:2.0:1788972592.407417:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972592.407421:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972592.407424:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525908 00000400:00000010:2.0:1788972592.407426:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525908. 00000100:00000200:2.0:1788972592.407432:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189622656/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000001:02000400:2.0:1788972592.409683:0:133170:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 612: lnet_legacy2yaml: malformed IP pattern ========================================================== 12:49:50 (1788972590) 00000400:00000200:3.0:1788972592.419341:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419355:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972592.419368:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419375:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419384:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419389:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a900 00000400:00000200:3.0:1788972592.419397:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419419:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a35 [1] + 224 00000400:00000200:3.0:1788972592.419436:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972592.419442:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972592.419479:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189622528/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972592.419517:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419521:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972592.419526:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419529:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419532:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419536:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781a880 00000400:00000200:3.0:1788972592.419540:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419546:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a3d [1] + 224 00000400:00000200:3.0:1788972592.419553:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972592.419557:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972592.419564:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1c00 x1875872189622400/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972592.419610:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419614:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972592.419618:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419621:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419625:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419628:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781a980 00000400:00000200:3.0:1788972592.419633:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972592.419637:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a45 [1] + 224 00000400:00000200:3.0:1788972592.419647:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972592.419651:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972592.419659:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189622656/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972592.419983:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1db58. 00000400:00000200:1.0:1788972592.419991:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:1.0:1788972592.419993:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788972592.420001:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1c00 x1875872189622400/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972592.420010:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1c00 x1875872189622400/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972592.420041:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d268. 00000400:00000200:1.0:1788972592.420044:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972592.420045:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972592.420050:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189622656/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972592.420057:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189622656/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972592.420074:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d630. 00000400:00000200:1.0:1788972592.420076:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18 00000400:00000010:1.0:1788972592.420078:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972592.420082:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189622528/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972592.420091:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189622528/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972608 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972592.420125:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03ddc00. 02000000:00000010:1.0:1788972592.420132:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff100. 00000100:00000010:1.0:1788972592.420138:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972592.420143:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd400. 02000000:00000010:1.0:1788972592.420146:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff400. 00000100:00000010:1.0:1788972592.420148:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972592.420151:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:1.0:1788972592.420154:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffc00. 00000100:00000010:1.0:1788972592.420156:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 00000100:00000010:2.0:1788972593.426798:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106600. 00000100:00000010:3.0:1788972593.428399:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106660. 00000100:00000010:0.0:1788972597.524629:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972597.524658:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972597.524697:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972597.524703:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7d00. 00000100:00000010:0.0:1788972597.524709:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972597.524714:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7300. 02000000:00000010:0.0:1788972597.524925:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835400. 00000400:00000200:0.0:1788972597.524937:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972597.524942:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972597.524951:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8990. 00000100:00000200:0.0:1788972597.524985:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622784, portal 25 00000400:00000010:0.0:1788972597.524993:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972597.524998:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189622784, offset 0 00000400:00000200:0.0:1788972597.525010:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972597.525021:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972597.525030:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972597.525036:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972597.525041:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972597.525045:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972597.525048:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972597.525052:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [862:862:256:4294967295] 192.168.203.150@tcp LPNI seq info [857:857:8:4294967295] 00000400:00000200:0.0:1788972597.525057:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972597.525068:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972597.525075:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972597.525080:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7b00. 00000800:00000200:0.0:1788972597.525087:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972597.525095:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972597.525100:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972597.525114:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972597.525119:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49de0. 00000100:00000010:0.0:1788972597.525132:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e496c0. 02000000:00000010:0.0:1788972597.525142:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835800. 00000400:00000200:0.0:1788972597.525145:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972597.525147:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4210. 00000400:00000010:0.0:1788972597.525151:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972597.525153:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623040, portal 10 00000400:00000010:0.0:1788972597.525159:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972597.525161:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189623040, offset 0 00000400:00000200:0.0:1788972597.525165:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972597.525168:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972597.525171:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972597.525180:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972597.525183:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972597.525185:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972597.525188:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972597.525191:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [863:863:255:4294967295] 192.168.203.150@tcp LPNI seq info [858:858:7:4294967295] 00000400:00000200:0.0:1788972597.525194:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972597.525201:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972597.525206:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972597.525209:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7f00. 00000800:00000200:0.0:1788972597.525213:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972597.525217:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972597.525220:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972597.525227:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972597.525247:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600. 00000400:00000200:0.0:1788972597.525251:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972597.525253:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4d68. 00000400:00000010:0.0:1788972597.525256:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972597.525259:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189622912, portal 10 00000400:00000010:0.0:1788972597.525263:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788972597.525265:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189622912, offset 0 00000400:00000200:0.0:1788972597.525280:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972597.525283:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972597.525286:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972597.525289:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972597.525291:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972597.525293:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972597.525295:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972597.525298:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [864:864:254:4294967295] 192.168.203.150@tcp LPNI seq info [859:859:6:4294967295] 00000400:00000200:0.0:1788972597.525301:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972597.525306:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972597.525317:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972597.525321:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7000. 00000800:00000200:0.0:1788972597.525324:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972597.525327:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972597.525330:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972597.525335:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972597.527986:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972597.528000:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7b00. 00000400:00000200:2.0:1788972597.528008:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972597.528029:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972597.528038:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28 00000400:00000010:2.0:1788972597.528041:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28. 00000100:00000200:2.0:1788972597.528055:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189622784/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972597.528086:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972597.528089:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7f00. 00000400:00000200:2.0:1788972597.528092:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972597.528095:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972597.528098:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a87f8 00000400:00000010:2.0:1788972597.528100:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a87f8. 00000100:00000200:2.0:1788972597.528106:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189623040/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972597.532763:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972597.532776:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7000. 00000400:00000200:2.0:1788972597.532785:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972597.532796:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972597.532804:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:2.0:1788972597.532807:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:2.0:1788972597.532928:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189622912/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972597.538107:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538120:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972597.538131:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538137:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538144:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538149:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781ab00 00000400:00000200:2.0:1788972597.538156:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538164:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a55 [1] + 224 00000400:00000200:2.0:1788972597.538179:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972597.538184:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972597.538213:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189623040/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972597.538254:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538257:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972597.538262:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538264:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538641:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538648:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781aa80 00000400:00000200:2.0:1788972597.538652:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538657:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a5d [1] + 224 00000400:00000200:2.0:1788972597.538666:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972597.538670:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972597.538678:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189622912/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972597.538695:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538699:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972597.538703:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538706:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538709:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538712:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781aa00 00000400:00000200:2.0:1788972597.538715:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972597.538720:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a4d [1] + 224 00000400:00000200:2.0:1788972597.538747:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972597.538751:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972597.538759:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189622784/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972597.538971:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4d68. 00000400:00000200:0.0:1788972597.538977:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:0.0:1788972597.538980:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972597.538988:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189622912/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972597.538997:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189622912/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972597.539034:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972597.539037:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8990 00000400:00000010:0.0:1788972597.539039:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8990. 00000100:00000200:0.0:1788972597.539043:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189622784/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972597.539051:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189622784/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972597.539062:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4210. 00000400:00000200:0.0:1788972597.539064:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550 00000400:00000010:0.0:1788972597.539066:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972597.539070:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189623040/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972597.539078:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189623040/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972597.539096:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600. 02000000:00000010:0.0:1788972597.539105:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7d00. 00000100:00000010:0.0:1788972597.539118:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972597.539123:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835400. 02000000:00000010:0.0:1788972597.539126:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972597.539127:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972597.539130:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835800. 02000000:00000010:0.0:1788972597.539132:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7300. 00000100:00000010:0.0:1788972597.539134:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 00000100:00000010:2.0:1788972598.545972:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49de0. 00000100:00000010:3.0:1788972598.546884:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e496c0. 00000001:02000400:3.0:1788972599.218276:0:133749:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 613: lnet_legacy2yaml: multiple IP patterns in single rule ========================================================== 12:49:57 (1788972597) 00000100:00000010:2.0:1788972602.646647:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972602.646669:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387800. 00000100:00000010:2.0:1788972602.646710:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100. 02000000:00000010:2.0:1788972602.646715:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387f00. 00000100:00000010:2.0:1788972602.646720:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ea00. 02000000:00000010:2.0:1788972602.646723:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387900. 02000000:00000010:2.0:1788972602.646754:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cee000. 00000400:00000200:2.0:1788972602.646762:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972602.646766:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972602.646772:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972602.646778:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623168, portal 25 00000400:00000010:2.0:1788972602.646782:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972602.646785:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189623168, offset 0 00000400:00000200:2.0:1788972602.646793:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972602.646801:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.646808:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972602.653362:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972602.653375:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972602.653389:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972602.653393:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972602.653397:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [865:865:256:4294967295] 192.168.203.150@tcp LPNI seq info [860:860:8:4294967295] 00000400:00000200:2.0:1788972602.653403:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.653412:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972602.653419:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972602.653427:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387d00. 00000800:00000200:2.0:1788972602.653434:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972602.653440:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972602.653445:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972602.653465:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972602.653567:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972602.653572:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387d00. 00000400:00000200:2.0:1788972602.653578:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972602.653583:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972602.653596:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788972602.653599:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972602.653610:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189623168/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972602.653655:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef200. 00000400:00000200:2.0:1788972602.653662:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972602.653667:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000010:2.0:1788972602.653672:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972602.653678:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623296, portal 10 00000400:00000010:2.0:1788972602.653686:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972602.653689:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189623296, offset 0 00000400:00000200:2.0:1788972602.653697:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972602.653704:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.653711:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972602.653714:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972602.653717:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972602.653720:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972602.653724:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972602.653727:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [866:866:256:4294967295] 192.168.203.150@tcp LPNI seq info [861:861:8:4294967295] 00000400:00000200:2.0:1788972602.653732:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.653737:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972602.653742:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972602.653746:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387d00. 00000800:00000200:2.0:1788972602.653750:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972602.653755:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972602.653758:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972602.653771:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972602.653776:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c784e0. 00000100:00000010:2.0:1788972602.653787:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78cc0. 00000800:00000200:2.0:1788972602.653849:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972602.653858:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387d00. 00000400:00000200:2.0:1788972602.653861:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972602.653865:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972602.653869:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972602.653871:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972602.653877:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f100 x1875872189623296/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972602.653908:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4627200. 00000400:00000200:2.0:1788972602.653913:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972602.653916:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580. 00000400:00000010:2.0:1788972602.653919:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972602.653922:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623424, portal 10 00000400:00000010:2.0:1788972602.653927:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972602.653929:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189623424, offset 0 00000400:00000200:2.0:1788972602.653933:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972602.653937:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.653940:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972602.653943:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972602.653945:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972602.653948:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972602.653950:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972602.653953:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [867:867:256:4294967295] 192.168.203.150@tcp LPNI seq info [862:862:8:4294967295] 00000400:00000200:2.0:1788972602.653957:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.653963:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972602.653968:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972602.653971:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387d00. 00000800:00000200:2.0:1788972602.653976:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972602.653980:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972602.653983:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972602.653994:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972602.654007:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972602.654015:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387d00. 00000400:00000200:2.0:1788972602.654018:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972602.654022:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972602.654026:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788972602.654028:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972602.654034:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ea00 x1875872189623424/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972602.655498:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.655507:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972602.655513:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.655516:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.655521:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.655524:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781ab80 00000400:00000200:2.0:1788972602.655529:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.655534:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a65 [1] + 224 00000400:00000200:2.0:1788972602.655547:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972602.655551:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972602.655562:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189623168/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972602.655601:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:2.0:1788972602.655605:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972602.655607:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972602.655612:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189623168/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972602.655620:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189623168/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972602.655646:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cee000. 02000000:00000010:2.0:1788972602.655652:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387800. 00000100:00000010:2.0:1788972602.655656:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000400:00000200:2.0:1788972602.682356:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682377:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972602.682397:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682403:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682410:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682414:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781ac00 00000400:00000200:2.0:1788972602.682421:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682427:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a75 [1] + 224 00000400:00000200:2.0:1788972602.682444:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972602.682450:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972602.682466:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f100 x1875872189623296/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972602.682500:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682508:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972602.682514:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682516:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682519:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682521:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781ac80 00000400:00000200:2.0:1788972602.682524:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972602.682529:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a85 [1] + 224 00000400:00000200:2.0:1788972602.682540:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972602.682544:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972602.682550:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ea00 x1875872189623424/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972602.682599:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:2.0:1788972602.682604:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:2.0:1788972602.682607:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972602.682612:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f100 x1875872189623296/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972602.682619:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f100 x1875872189623296/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972602.682665:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef200. 02000000:00000010:2.0:1788972602.682672:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387f00. 00000100:00000010:2.0:1788972602.682679:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100. 00000400:00000010:2.0:1788972602.682703:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580. 00000400:00000200:2.0:1788972602.682707:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972602.682708:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972602.682713:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ea00 x1875872189623424/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972602.682722:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ea00 x1875872189623424/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972618 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972602.682735:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4627200. 02000000:00000010:2.0:1788972602.682741:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387900. 00000100:00000010:2.0:1788972602.682744:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ea00. 00000100:00000010:3.0:1788972603.665897:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78cc0. 00000100:00000010:2.0:1788972603.665968:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c784e0. 00000001:02000400:2.0:1788972606.154782:0:134329:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 614: lnet_legacy2yaml: interfaces enumeration with multiple IP ranges ========================================================== 12:50:05 (1788972605) 00000100:00000010:1.0:1788972607.762747:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972607.762769:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 00000100:00000010:1.0:1788972607.762796:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972607.762800:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:1.0:1788972607.762806:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972607.762810:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63200. 02000000:00000010:1.0:1788972607.762885:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8834800. 00000400:00000200:1.0:1788972607.762895:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972607.762899:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d630. 00000400:00000010:1.0:1788972607.762905:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972607.762912:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623552, portal 25 00000400:00000010:1.0:1788972607.762917:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972607.762921:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189623552, offset 0 00000400:00000200:1.0:1788972607.762932:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972607.762940:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972607.762948:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972607.762953:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972607.762957:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972607.762961:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972607.762964:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972607.762967:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [868:868:256:4294967295] 192.168.203.150@tcp LPNI seq info [863:863:8:4294967295] 00000400:00000200:1.0:1788972607.762978:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972607.762988:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972607.762994:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972607.762998:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00. 00000800:00000200:1.0:1788972607.763004:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972607.763012:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972607.763016:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972607.763036:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972607.763042:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c41064e0. 00000100:00000010:1.0:1788972607.763050:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106ae0. 02000000:00000010:1.0:1788972607.763060:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835000. 00000400:00000200:1.0:1788972607.763064:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972607.763066:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d268. 00000400:00000010:1.0:1788972607.763069:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788972607.763072:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623808, portal 10 00000400:00000010:1.0:1788972607.763077:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525ee0. 00000100:00000200:1.0:1788972607.763079:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189623808, offset 0 00000400:00000200:1.0:1788972607.763083:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972607.763087:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972607.763090:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972607.763092:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972607.763094:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972607.763096:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972607.763098:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972607.763101:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [869:869:255:4294967295] 192.168.203.150@tcp LPNI seq info [864:864:7:4294967295] 00000400:00000200:1.0:1788972607.763104:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972607.763109:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972607.763113:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972607.763116:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63100. 00000800:00000200:1.0:1788972607.763119:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972607.763126:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000200:1.0:1788972607.763129:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000010:2.0:1788972607.763130:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00. 00000800:00000200:1.0:1788972607.763131:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972607.763134:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972607.763136:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.763138:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972607.763143:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:2.0:1788972607.763145:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 02000000:00000010:1.0:1788972607.763152:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835e00. 00000100:00000200:2.0:1788972607.763153:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189623552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972623 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972607.763155:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972607.763158:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1db58. 00000400:00000010:1.0:1788972607.763161:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972607.763163:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623680, portal 10 00000400:00000010:1.0:1788972607.763167:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525440. 00000100:00000200:1.0:1788972607.763169:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189623680, offset 0 00000400:00000200:1.0:1788972607.763173:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000800:00000200:2.0:1788972607.763175:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1788972607.763176:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:2.0:1788972607.763177:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63100. 00000400:00000200:2.0:1788972607.763179:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972607.763180:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972607.763182:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:1.0:1788972607.763182:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972607.763185:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525ee0 00000400:00000200:1.0:1788972607.763185:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000010:2.0:1788972607.763186:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525ee0. 00000400:00000200:1.0:1788972607.763187:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972607.763189:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972607.763192:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [870:870:255:4294967295] 192.168.203.150@tcp LPNI seq info [865:865:7:4294967295] 00000400:00000200:1.0:1788972607.763195:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:2.0:1788972607.763196:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1c00 x1875872189623808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972607.763202:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972607.763206:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972607.763208:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63700. 00000800:00000200:1.0:1788972607.763212:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972607.763216:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972607.763218:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972607.763229:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972607.763258:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972607.763261:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63700. 00000400:00000200:2.0:1788972607.763263:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972607.763267:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972607.763270:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525440 00000400:00000010:2.0:1788972607.763271:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525440. 00000100:00000200:2.0:1788972607.763275:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189623680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972607.771492:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.771502:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972607.771512:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.771516:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.771520:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.771524:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781ad00 00000400:00000200:2.0:1788972607.771530:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.771536:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a95 [1] + 224 00000400:00000200:2.0:1788972607.771550:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972607.771554:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972607.771567:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189623552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972607.771887:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d630. 00000400:00000200:0.0:1788972607.771893:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18 00000400:00000010:0.0:1788972607.771895:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18. 00000100:00000200:0.0:1788972607.771902:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189623552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972607.771910:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189623552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972607.771936:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8834800. 02000000:00000010:0.0:1788972607.771942:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:0.0:1788972607.771946:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 00000400:00000200:2.0:1788972607.777765:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777776:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972607.777785:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777789:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777794:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777799:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781ad80 00000400:00000200:2.0:1788972607.777804:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777809:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2aa5 [1] + 224 00000400:00000200:2.0:1788972607.777885:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972607.777890:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972607.777913:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189623680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972607.777987:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777991:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972607.777995:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777997:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.777999:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.778002:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781ae00 00000400:00000200:2.0:1788972607.778005:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972607.778008:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2a9d [1] + 224 00000400:00000200:2.0:1788972607.778029:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972607.778033:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972607.778040:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1c00 x1875872189623808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972607.781050:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1db58. 00000400:00000200:0.0:1788972607.781059:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:0.0:1788972607.781062:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:0.0:1788972607.781075:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189623680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972607.781087:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189623680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972607.781117:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d268. 00000400:00000200:0.0:1788972607.781119:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:0.0:1788972607.781121:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:0.0:1788972607.781126:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1c00 x1875872189623808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972607.781134:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1c00 x1875872189623808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972623 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972607.781157:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835e00. 02000000:00000010:0.0:1788972607.781163:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:0.0:1788972607.781167:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 02000000:00000010:0.0:1788972607.781170:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835000. 02000000:00000010:0.0:1788972607.781172:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63200. 00000100:00000010:0.0:1788972607.781174:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00. 00000100:00000010:2.0:1788972608.786139:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c41064e0. 00000100:00000010:3.0:1788972608.786642:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106ae0. 00000100:00000010:0.0:1788972612.890884:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972612.891146:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7500. 00000100:00000010:0.0:1788972612.891192:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972612.891198:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7400. 00000100:00000010:0.0:1788972612.891205:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972612.891209:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c52b7600. 02000000:00000010:0.0:1788972612.891253:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a000. 00000400:00000200:0.0:1788972612.891266:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972612.891271:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4210. 00000400:00000010:0.0:1788972612.891279:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550. 00000100:00000200:0.0:1788972612.891289:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189623936, portal 25 00000400:00000010:0.0:1788972612.891295:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8990. 00000100:00000200:0.0:1788972612.891300:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189623936, offset 0 00000400:00000200:0.0:1788972612.891328:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972612.891339:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972612.891348:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972612.891355:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972612.891359:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972612.891364:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972612.891368:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972612.891372:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [871:871:256:4294967295] 192.168.203.150@tcp LPNI seq info [866:866:8:4294967295] 00000400:00000200:0.0:1788972612.891378:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972612.891390:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972612.891396:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972612.891419:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7100. 00000800:00000200:0.0:1788972612.891428:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972612.891437:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972612.891444:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972612.891857:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972612.891865:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e498a0. 00000100:00000010:0.0:1788972612.891878:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49000. 02000000:00000010:0.0:1788972612.891894:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b200. 00000400:00000200:0.0:1788972612.891898:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972612.891902:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972612.891906:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972612.891920:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624192, portal 10 00000400:00000010:0.0:1788972612.891929:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8cc0. 00000100:00000200:0.0:1788972612.891932:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189624192, offset 0 00000400:00000200:0.0:1788972612.891938:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972612.891943:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972612.891947:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972612.891951:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972612.891953:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972612.891955:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972612.891958:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972612.891961:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [872:872:255:4294967295] 192.168.203.150@tcp LPNI seq info [867:867:7:4294967295] 00000400:00000200:0.0:1788972612.891966:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972612.891973:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972612.891978:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972612.891982:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7a00. 00000800:00000200:0.0:1788972612.891986:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972612.891991:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972612.891995:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972612.892001:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972612.892177:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800. 00000400:00000200:0.0:1788972612.892183:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972612.892186:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4d68. 00000400:00000010:0.0:1788972612.892190:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8bb0. 00000100:00000200:0.0:1788972612.892193:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624064, portal 10 00000400:00000010:0.0:1788972612.892200:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a82a8. 00000100:00000200:0.0:1788972612.892202:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189624064, offset 0 00000400:00000200:0.0:1788972612.892208:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972612.892212:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972612.892215:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972612.892218:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972612.892221:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972612.892223:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972612.892225:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972612.892228:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [873:873:254:4294967295] 192.168.203.150@tcp LPNI seq info [868:868:6:4294967295] 00000400:00000200:0.0:1788972612.892232:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972612.892238:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972612.892243:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972612.892247:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7300. 00000800:00000200:0.0:1788972612.892251:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972612.892256:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972612.892258:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972612.892264:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972612.892990:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972612.892998:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7100. 00000400:00000200:2.0:1788972612.893004:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972612.893012:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972612.893098:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8990 00000400:00000010:2.0:1788972612.893102:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8990. 00000100:00000200:2.0:1788972612.893115:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189623936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972628 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972612.893150:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972612.893153:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7a00. 00000400:00000200:2.0:1788972612.893156:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972612.893159:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972612.893162:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8cc0 00000400:00000010:2.0:1788972612.893178:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8cc0. 00000100:00000200:2.0:1788972612.893185:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189624192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972612.893275:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972612.893278:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7300. 00000400:00000200:2.0:1788972612.893282:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972612.893286:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972612.893289:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a82a8 00000400:00000010:2.0:1788972612.893291:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a82a8. 00000100:00000200:2.0:1788972612.893296:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189624064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972612.903995:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904009:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972612.904021:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904026:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904033:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904037:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781ae80 00000400:00000200:2.0:1788972612.904043:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904050:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2aad [1] + 224 00000400:00000200:2.0:1788972612.904063:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972612.904068:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972612.904086:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189623936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972612.904126:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904129:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972612.904133:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904135:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904138:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904140:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781af80 00000400:00000200:2.0:1788972612.904144:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.904159:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2ab5 [1] + 224 00000400:00000200:2.0:1788972612.904172:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972612.904175:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972612.904182:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189624192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972612.905079:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4210. 00000400:00000200:1.0:1788972612.905092:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550 00000400:00000010:1.0:1788972612.905094:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550. 00000100:00000200:1.0:1788972612.905105:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189623936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972612.905132:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189623936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972612.905165:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:1.0:1788972612.905169:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:1.0:1788972612.905172:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:1.0:1788972612.905178:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189624192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972612.905189:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189624192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972612.905213:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a000. 02000000:00000010:1.0:1788972612.905332:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7500. 00000100:00000010:1.0:1788972612.905339:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 02000000:00000010:1.0:1788972612.905343:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b200. 02000000:00000010:1.0:1788972612.905421:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7600. 00000100:00000010:1.0:1788972612.905425:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 00000400:00000200:2.0:1788972612.910979:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.910991:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972612.911001:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.911008:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.911014:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.911018:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781af00 00000400:00000200:2.0:1788972612.911024:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972612.911031:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2abd [1] + 224 00000400:00000200:2.0:1788972612.911049:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972612.911056:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972612.911677:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189624064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972612.919909:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4d68. 00000400:00000200:1.0:1788972612.919947:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8bb0 00000400:00000010:1.0:1788972612.919951:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8bb0. 00000100:00000200:1.0:1788972612.919971:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189624064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972612.919983:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189624064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972628 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972612.920026:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800. 02000000:00000010:1.0:1788972612.920035:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c52b7400. 00000100:00000010:1.0:1788972612.920040:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000001:02000400:0.0:1788972613.404302:0:134909:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 615: lnet_legacy2yaml: modprobe configuration file input ========================================================== 12:50:12 (1788972612) 00000100:00000010:3.0:1788972613.905893:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49000. 00000100:00000010:2.0:1788972613.908567:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e498a0. 00000100:00000010:1.0:1788972618.017048:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972618.017079:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63200. 00000100:00000010:1.0:1788972618.017118:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:0.0:1788972618.017248:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c46ca800. 00000400:00000200:0.0:1788972618.017263:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972618.017268:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4d10. 00000400:00000010:0.0:1788972618.017291:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972618.017301:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624320, portal 25 00000400:00000010:0.0:1788972618.017308:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972618.017319:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189624320, offset 0 00000400:00000200:0.0:1788972618.017329:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972618.017339:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972618.017349:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972618.017355:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972618.017359:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972618.017363:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972618.017366:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972618.017371:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [874:874:256:4294967295] 192.168.203.150@tcp LPNI seq info [869:869:8:4294967295] 00000400:00000200:0.0:1788972618.017375:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972618.017386:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972618.017393:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972618.017400:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c52b7c00. 00000800:00000200:0.0:1788972618.017409:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972618.017417:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972618.017423:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c52b7c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972618.017445:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972618.017452:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49660. 00000100:00000010:0.0:1788972618.017468:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4e49360. 02000000:00000010:1.0:1788972618.017607:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 00000100:00000010:1.0:1788972618.017630:0:118499:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972618.017637:0:118499:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 02000000:00000010:1.0:1788972618.017665:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef200. 00000400:00000200:1.0:1788972618.017670:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972618.017674:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d790. 00000400:00000010:1.0:1788972618.017678:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972618.017681:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624448, portal 10 00000400:00000010:1.0:1788972618.017688:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525aa0. 00000100:00000200:1.0:1788972618.017691:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189624448, offset 0 00000400:00000200:1.0:1788972618.017697:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972618.017701:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972618.017705:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972618.017708:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972618.017710:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972618.017712:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972618.017714:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972618.017717:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [875:875:255:4294967295] 192.168.203.150@tcp LPNI seq info [870:870:7:4294967295] 00000400:00000200:1.0:1788972618.017720:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972618.017726:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972618.017731:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972618.017735:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d207d900. 00000800:00000200:1.0:1788972618.017739:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972618.017743:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972618.017746:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d207d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972618.017753:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972618.017767:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cefa00. 00000400:00000200:1.0:1788972618.017770:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972618.017773:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d058. 00000400:00000010:1.0:1788972618.017777:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75257f8. 00000100:00000200:1.0:1788972618.017779:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624576, portal 10 00000400:00000010:1.0:1788972618.017791:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75253b8. 00000100:00000200:1.0:1788972618.017794:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189624576, offset 0 00000400:00000200:1.0:1788972618.017799:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972618.017803:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972618.017807:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972618.017809:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972618.017833:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972618.017836:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972618.017839:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972618.017842:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [876:876:254:4294967295] 192.168.203.150@tcp LPNI seq info [871:871:6:4294967295] 00000400:00000200:1.0:1788972618.017846:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972618.017851:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972618.017855:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972618.017859:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d207d800. 00000800:00000200:1.0:1788972618.017862:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972618.017867:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972618.017869:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d207d800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972618.017874:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972618.018402:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972618.018409:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c52b7c00. 00000400:00000200:2.0:1788972618.018415:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972618.018423:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972618.018430:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28 00000400:00000010:2.0:1788972618.018433:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28. 00000100:00000200:2.0:1788972618.018445:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1c00 x1875872189624320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972618.018477:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972618.018479:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d207d900. 00000400:00000200:2.0:1788972618.018483:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972618.018486:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972618.018498:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525aa0 00000400:00000010:2.0:1788972618.018500:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525aa0. 00000100:00000200:2.0:1788972618.018584:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189624448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972618.018671:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972618.018674:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d207d800. 00000400:00000200:2.0:1788972618.018678:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972618.018681:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972618.018684:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75253b8 00000400:00000010:2.0:1788972618.018686:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75253b8. 00000100:00000200:2.0:1788972618.018690:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189624576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972618.024121:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024621:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972618.024635:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024641:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024650:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024654:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781b080 00000400:00000200:2.0:1788972618.024661:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024668:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2acd [1] + 224 00000400:00000200:2.0:1788972618.024685:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972618.024690:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972618.024709:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189624448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972618.024753:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024758:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972618.024762:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024764:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024767:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024770:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781b100 00000400:00000200:2.0:1788972618.024782:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.024787:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2ad5 [1] + 224 00000400:00000200:2.0:1788972618.024796:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972618.024799:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972618.024807:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189624576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972618.024995:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d790. 00000400:00000200:1.0:1788972618.025003:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:1.0:1788972618.025006:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972618.025014:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189624448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972618.025026:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189624448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972618.025068:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef200. 02000000:00000010:1.0:1788972618.025076:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:1.0:1788972618.025082:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000400:00000200:2.0:1788972618.025803:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.025810:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972618.025833:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.025837:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.025840:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.025844:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781b000 00000400:00000200:2.0:1788972618.025848:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972618.025854:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2ac5 [1] + 224 00000400:00000200:2.0:1788972618.025871:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972618.025876:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972618.025886:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1c00 x1875872189624320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972618.026798:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d058. 00000400:00000200:1.0:1788972618.026807:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75257f8 00000400:00000010:1.0:1788972618.026851:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75257f8. 00000100:00000200:1.0:1788972618.026861:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189624576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972618.026872:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189624576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972618.026905:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cefa00. 02000000:00000010:1.0:1788972618.026911:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:1.0:1788972618.026916:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 00000400:00000010:0.0:1788972618.029508:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4d10. 00000400:00000200:0.0:1788972618.029535:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a87f8 00000400:00000010:0.0:1788972618.029538:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972618.029566:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1c00 x1875872189624320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972618.029580:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1c00 x1875872189624320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972634 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972618.029618:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c46ca800. 02000000:00000010:0.0:1788972618.029626:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63200. 00000100:00000010:0.0:1788972618.029632:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00. 00000100:00000010:3.0:1788972619.026107:0:109114:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49360. 00000100:00000010:2.0:1788972619.027017:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4e49660. 00000001:02000400:3.0:1788972620.849311:0:135491:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 616: lnet_legacy2yaml: combining modprobe file with explicit ip2nets file ========================================================== 12:50:19 (1788972619) 00000100:00000010:1.0:1788972623.125252:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972623.125277:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63200. 00000100:00000010:1.0:1788972623.125309:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972623.125314:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 00000100:00000010:1.0:1788972623.125321:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80. 02000000:00000010:1.0:1788972623.125324:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 02000000:00000010:1.0:1788972623.125354:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835800. 00000400:00000200:1.0:1788972623.125363:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972623.125437:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d058. 00000400:00000010:1.0:1788972623.125446:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75257f8. 00000100:00000200:1.0:1788972623.125453:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624704, portal 25 00000400:00000010:1.0:1788972623.125458:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972623.125461:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189624704, offset 0 00000400:00000200:1.0:1788972623.125470:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972623.125479:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972623.125486:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972623.125490:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972623.125494:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972623.125498:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972623.125501:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972623.125504:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [877:877:256:4294967295] 192.168.203.150@tcp LPNI seq info [872:872:8:4294967295] 00000400:00000200:1.0:1788972623.125508:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972623.125517:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972623.125530:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972623.125537:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5721600. 00000800:00000200:1.0:1788972623.125544:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972623.125551:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972623.125555:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5721600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972623.125569:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972623.125573:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106480. 00000100:00000010:1.0:1788972623.125584:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c4106e40. 02000000:00000010:1.0:1788972623.125595:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8834600. 00000400:00000200:1.0:1788972623.125598:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972623.125601:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d790. 00000400:00000010:1.0:1788972623.125604:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525bb0. 00000100:00000200:1.0:1788972623.125608:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624960, portal 10 00000400:00000010:1.0:1788972623.125613:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75254c8. 00000100:00000200:1.0:1788972623.125615:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189624960, offset 0 00000400:00000200:1.0:1788972623.125619:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972623.125622:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972623.125626:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972623.125628:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972623.125631:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972623.125633:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972623.125634:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972623.125637:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [878:878:255:4294967295] 192.168.203.150@tcp LPNI seq info [873:873:7:4294967295] 00000400:00000200:1.0:1788972623.125640:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972623.125646:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972623.125649:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972623.125652:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5721c00. 00000800:00000200:1.0:1788972623.125656:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972623.125659:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972623.125661:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5721c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972623.125672:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972623.125674:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972623.125679:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5721600. 00000400:00000200:2.0:1788972623.125684:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972623.125689:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 02000000:00000010:1.0:1788972623.125689:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8834800. 00000400:00000200:1.0:1788972623.125692:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972623.125694:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d4d0. 00000400:00000200:2.0:1788972623.125695:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:2.0:1788972623.125697:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000400:00000010:1.0:1788972623.125697:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788972623.125700:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189624832, portal 10 00000400:00000010:1.0:1788972623.125704:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75255d8. 00000100:00000200:1.0:1788972623.125706:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189624832, offset 0 00000100:00000200:2.0:1788972623.125707:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189624704/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972623.125710:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972623.125713:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972623.125717:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972623.125719:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972623.125721:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972623.125723:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972623.125725:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972623.125727:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [879:879:255:4294967295] 192.168.203.150@tcp LPNI seq info [874:874:7:4294967295] 00000400:00000200:1.0:1788972623.125731:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972623.125736:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972623.125740:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972623.125743:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5721f00. 00000800:00000200:1.0:1788972623.125746:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972623.125749:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972623.125751:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5721f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972623.125755:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972623.125781:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972623.125784:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5721c00. 00000400:00000200:2.0:1788972623.125787:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972623.125795:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972623.125799:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75254c8 00000400:00000010:2.0:1788972623.125800:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75254c8. 00000100:00000200:2.0:1788972623.125805:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1f80 x1875872189624960/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972623.125840:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972623.125842:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5721f00. 00000400:00000200:2.0:1788972623.125845:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972623.125848:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972623.125850:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75255d8 00000400:00000010:2.0:1788972623.125852:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75255d8. 00000100:00000200:2.0:1788972623.125856:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189624832/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972623.131315:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131324:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972623.131333:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131338:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131343:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131346:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781b180 00000400:00000200:3.0:1788972623.131351:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131380:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2add [1] + 224 00000400:00000200:3.0:1788972623.131393:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972623.131398:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972623.131412:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189624704/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972623.131438:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131442:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972623.131446:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131448:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131451:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131454:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781b280 00000400:00000200:3.0:1788972623.131458:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131462:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2ae5 [1] + 224 00000400:00000200:3.0:1788972623.131468:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972623.131582:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972623.131592:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1f80 x1875872189624960/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972623.131610:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131613:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972623.131618:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131621:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131624:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131627:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781b200 00000400:00000200:3.0:1788972623.131630:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972623.131634:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2aed [1] + 224 00000400:00000200:3.0:1788972623.131642:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972623.131645:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972623.131651:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189624832/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972623.131709:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d4d0. 00000400:00000200:1.0:1788972623.131714:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:1.0:1788972623.131717:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788972623.131723:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189624832/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972623.131732:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189624832/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972623.131772:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d058. 00000400:00000200:1.0:1788972623.131775:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75257f8 00000400:00000010:1.0:1788972623.131777:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75257f8. 00000100:00000200:1.0:1788972623.131782:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189624704/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972623.131790:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189624704/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972623.131800:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d790. 00000400:00000200:1.0:1788972623.131802:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525bb0 00000400:00000010:1.0:1788972623.131803:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525bb0. 00000100:00000200:1.0:1788972623.131807:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1f80 x1875872189624960/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972623.131832:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1f80 x1875872189624960/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972639 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972623.131848:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8834800. 02000000:00000010:1.0:1788972623.131854:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:1.0:1788972623.131858:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972623.131862:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835800. 02000000:00000010:1.0:1788972623.131866:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63200. 00000100:00000010:1.0:1788972623.131868:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972623.131870:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8834600. 02000000:00000010:1.0:1788972623.131872:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:1.0:1788972623.131874:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80. 00000100:00000010:2.0:1788972624.146273:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106480. 00000100:00000010:3.0:1788972624.146926:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c4106e40. 00000001:02000400:0.0:1788972628.114112:0:136073:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 617: lnet_legacy2yaml: round-trip tunables from LNet to modprobe to YAML ========================================================== 12:50:26 (1788972626) 00000400:00000010:1.0:1788972628.177968:0:102137:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at ffff9a73c381b000. 00000100:00000010:3.0:1788972628.245312:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788972628.245334:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bd00. 00000100:00000010:3.0:1788972628.245360:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788972628.245364:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bb00. 00000100:00000010:3.0:1788972628.245370:0:80315:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788972628.245373:0:80315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b700. 02000000:00000010:2.0:1788972628.245451:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd600. 00000400:00000200:2.0:1788972628.245461:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972628.245466:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126580. 00000400:00000010:2.0:1788972628.245473:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972628.245479:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189625088, portal 25 00000400:00000010:2.0:1788972628.245494:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972628.245497:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189625088, offset 0 00000400:00000200:2.0:1788972628.245506:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972628.245514:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972628.245523:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972628.245528:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972628.245532:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972628.245535:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972628.245538:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972628.245541:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [880:880:256:4294967295] 192.168.203.150@tcp LPNI seq info [875:875:8:4294967295] 00000400:00000200:2.0:1788972628.245546:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972628.245556:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972628.245570:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972628.245575:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387500. 00000800:00000200:2.0:1788972628.245581:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972628.245588:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972628.245592:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972628.245614:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972628.245619:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78300. 00000100:00000010:2.0:1788972628.245629:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7c78720. 02000000:00000010:2.0:1788972628.245639:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 00000400:00000200:2.0:1788972628.245643:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972628.245645:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126160. 00000400:00000010:2.0:1788972628.245649:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972628.245651:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189625344, portal 10 00000400:00000010:2.0:1788972628.245656:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972628.245658:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189625344, offset 0 00000400:00000200:2.0:1788972628.245662:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972628.245666:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972628.245670:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972628.245673:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972628.245675:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972628.245678:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972628.245680:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972628.245683:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [881:881:255:4294967295] 192.168.203.150@tcp LPNI seq info [876:876:7:4294967295] 00000400:00000200:2.0:1788972628.245687:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972628.245693:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972628.245697:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972628.245700:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387100. 00000800:00000200:2.0:1788972628.245704:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972628.245709:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972628.245712:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972628.245724:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972628.245742:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd400. 00000400:00000200:2.0:1788972628.245746:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972628.245749:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126dc0. 00000400:00000010:2.0:1788972628.245752:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788972628.245755:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189625216, portal 10 00000400:00000010:2.0:1788972628.245759:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972628.245762:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189625216, offset 0 00000400:00000200:2.0:1788972628.245767:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972628.245770:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972628.245773:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972628.245776:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972628.245779:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972628.245781:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972628.245783:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972628.245786:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [882:882:254:4294967295] 192.168.203.150@tcp LPNI seq info [877:877:6:4294967295] 00000400:00000200:2.0:1788972628.245790:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972628.245796:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972628.245800:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972628.245804:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387400. 00000800:00000200:2.0:1788972628.245808:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972628.245839:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972628.245842:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972628.245848:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972628.246024:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972628.246028:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387500. 00000400:00000200:3.0:1788972628.246035:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972628.246040:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972628.246045:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:3.0:1788972628.246047:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:3.0:1788972628.246055:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189625088/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972628.246078:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972628.246080:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387100. 00000400:00000200:3.0:1788972628.246083:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972628.246087:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972628.246089:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:3.0:1788972628.246091:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:3.0:1788972628.246095:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790e00 x1875872189625344/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972628.246177:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972628.246179:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387400. 00000400:00000200:3.0:1788972628.246182:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972628.246186:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972628.246189:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:3.0:1788972628.246191:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:3.0:1788972628.246195:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790a80 x1875872189625216/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972628.253226:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253235:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972628.253243:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253248:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253253:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253256:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781b400 00000400:00000200:3.0:1788972628.253261:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253266:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2b05 [1] + 224 00000400:00000200:3.0:1788972628.253277:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972628.253281:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972628.253294:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790e00 x1875872189625344/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972628.253324:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253327:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972628.253331:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253334:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253336:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253338:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188781b380 00000400:00000200:3.0:1788972628.253341:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253345:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2b15 [1] + 224 00000400:00000200:3.0:1788972628.253350:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972628.253353:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972628.253358:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790a80 x1875872189625216/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972628.253370:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253373:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972628.253376:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253378:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253388:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253390:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188781b300 00000400:00000200:3.0:1788972628.253394:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972628.253397:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2af5 [1] + 224 00000400:00000200:3.0:1788972628.253407:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972628.253410:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972628.253416:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189625088/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972628.253455:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126dc0. 00000400:00000200:3.0:1788972628.253460:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:3.0:1788972628.253463:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:3.0:1788972628.253468:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790a80 x1875872189625216/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972628.253477:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790a80 x1875872189625216/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972628.253509:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126580. 00000400:00000200:3.0:1788972628.253513:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:3.0:1788972628.253514:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:3.0:1788972628.253519:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189625088/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972628.253527:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189625088/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972628.253537:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126160. 00000400:00000200:3.0:1788972628.253540:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:3.0:1788972628.253541:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:3.0:1788972628.253545:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790e00 x1875872189625344/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972628.253552:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790e00 x1875872189625344/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972644 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972628.253582:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd400. 02000000:00000010:3.0:1788972628.253587:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bb00. 00000100:00000010:3.0:1788972628.253592:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788972628.253596:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd600. 02000000:00000010:3.0:1788972628.253598:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bd00. 00000100:00000010:3.0:1788972628.253600:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788972628.253603:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:3.0:1788972628.253605:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b700. 00000100:00000010:3.0:1788972628.253607:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790e00. 00000001:02000400:3.0:1788972628.372464:0:136087:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000100:00000010:2.0:1788972629.265910:0:131139:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78300. 00000100:00000010:3.0:1788972629.277474:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7c78720. 00000001:02000400:1.0:1788972629.898526:0:136215:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_617: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 Debug log: 4575 lines, 4575 kept, 0 dropped, 0 bad.