00000100:00000010:3.0F:1790701610.892090:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d180. 02000000:00000010:3.0:1790701610.892124:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441a00. 00000100:00000010:3.0:1790701610.892166:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074dc00. 02000000:00000010:3.0:1790701610.892171:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441000. 00000100:00000010:3.0:1790701610.892178:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ce00. 02000000:00000010:3.0:1790701610.892181:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441b00. 02000000:00000010:2.0F:1790701610.892267:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bb000. 00000400:00000200:2.0:1790701610.892284:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701610.892288:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2840. 00000400:00000010:2.0:1790701610.892294:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a550. 00000100:00000200:2.0:1790701610.892302:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537431808, portal 25 00000400:00000010:2.0:1790701610.892308:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6ac38. 00000100:00000200:2.0:1790701610.892311:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537431808, offset 0 00000400:00000200:2.0:1790701610.892321:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701610.892329:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701610.892337:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701610.892344:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701610.892349:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701610.892353:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701610.892356:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701610.892360:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [91:91:256:4294967295] 192.168.204.157@tcp LPNI seq info [91:91:8:4294967295] 00000400:00000200:2.0:1790701610.892365:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701610.892374:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701610.892381:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701610.892387:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4500. 00000800:00000200:2.0:1790701610.892394:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701610.892403:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701610.892407:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701610.892429:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:2.0:1790701610.892434:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce118f780. 00000100:00000010:2.0:1790701610.892444:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce118f6c0. 02000000:00000010:2.0:1790701610.892456:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bb200. 00000400:00000200:2.0:1790701610.892460:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701610.892462:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2058. 00000400:00000010:2.0:1790701610.892466:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a198. 00000100:00000200:2.0:1790701610.892469:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432064, portal 10 00000400:00000010:2.0:1790701610.892473:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6ab28. 00000100:00000200:2.0:1790701610.892475:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537432064, offset 0 00000400:00000200:2.0:1790701610.892485:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701610.892488:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701610.892491:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701610.892494:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701610.892497:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701610.892499:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701610.892501:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701610.892503:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [92:92:255:4294967295] 192.168.204.157@tcp LPNI seq info [92:92:7:4294967295] 00000400:00000200:2.0:1790701610.892506:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701610.892511:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701610.892515:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701610.892518:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4300. 00000800:00000200:2.0:1790701610.892522:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701610.892526:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701610.892529:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701610.892534:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:2.0:1790701610.892551:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83ba200. 00000400:00000200:2.0:1790701610.892554:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701610.892556:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b24d0. 00000400:00000010:2.0:1790701610.892560:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701610.892562:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537431936, portal 10 00000400:00000010:2.0:1790701610.892566:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a000. 00000100:00000200:2.0:1790701610.892568:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537431936, offset 0 00000400:00000200:2.0:1790701610.892572:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701610.892575:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701610.892578:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701610.892580:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701610.892583:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701610.892585:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701610.892587:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701610.892590:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [93:93:254:4294967295] 192.168.204.157@tcp LPNI seq info [93:93:6:4294967295] 00000400:00000200:2.0:1790701610.892599:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701610.892605:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701610.892609:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701610.892612:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4d00. 00000800:00000200:2.0:1790701610.892616:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701610.892620:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701610.892623:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701610.892628:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701610.892784:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701610.892788:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4500. 00000400:00000200:3.0:1790701610.892793:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701610.892798:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701610.892805:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6ac38 00000400:00000010:3.0:1790701610.892808:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6ac38. 00000100:00000200:3.0:1790701610.892866:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d180 x1877686537431808/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701610.892893:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701610.892896:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4300. 00000400:00000200:3.0:1790701610.892898:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701610.892901:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701610.892904:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6ab28 00000400:00000010:3.0:1790701610.892906:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6ab28. 00000100:00000200:3.0:1790701610.892910:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ce00 x1877686537432064/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701610.892994:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701610.892996:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4d00. 00000400:00000200:3.0:1790701610.893000:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701610.893003:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701610.893005:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a000 00000400:00000010:3.0:1790701610.893007:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a000. 00000100:00000200:3.0:1790701610.893011:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074dc00 x1877686537431936/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701610.897490:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897542:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701610.897563:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897569:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897575:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897579:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102c00 00000400:00000200:3.0:1790701610.897584:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897591:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x60d [1] + 224 00000400:00000200:3.0:1790701610.897604:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701610.897609:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701610.897625:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ce00 x1877686537432064/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701610.897659:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897662:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701610.897667:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897669:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897671:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897674:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102b80 00000400:00000200:3.0:1790701610.897677:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.897691:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x61d [1] + 224 00000400:00000200:3.0:1790701610.897699:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701610.897702:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701610.897708:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074dc00 x1877686537431936/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701610.898014:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b24d0. 00000400:00000200:2.0:1790701610.898020:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a330 00000400:00000010:2.0:1790701610.898022:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701610.898031:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074dc00 x1877686537431936/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701610.898040:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074dc00 x1877686537431936/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701610.898074:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2058. 00000400:00000200:2.0:1790701610.898076:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a198 00000400:00000010:2.0:1790701610.898078:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a198. 00000100:00000200:2.0:1790701610.898082:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ce00 x1877686537432064/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701610.898090:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ce00 x1877686537432064/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701610.898107:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83ba200. 02000000:00000010:2.0:1790701610.898112:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441000. 00000100:00000010:2.0:1790701610.898117:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074dc00. 02000000:00000010:2.0:1790701610.898120:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bb200. 02000000:00000010:2.0:1790701610.898122:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441b00. 00000100:00000010:2.0:1790701610.898124:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ce00. 00000400:00000200:3.0:1790701610.917265:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.917276:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701610.917285:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.917290:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.917297:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.917301:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102b00 00000400:00000200:3.0:1790701610.917306:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701610.917509:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x5fd [1] + 224 00000400:00000200:3.0:1790701610.917524:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701610.917528:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701610.917548:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d180 x1877686537431808/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701610.917602:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2840. 00000400:00000200:3.0:1790701610.917608:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a550 00000400:00000010:3.0:1790701610.917609:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a550. 00000100:00000200:3.0:1790701610.917617:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d180 x1877686537431808/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701610.917625:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d180 x1877686537431808/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701626 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701610.917658:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bb000. 02000000:00000010:3.0:1790701610.917664:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441a00. 00000100:00000010:3.0:1790701610.917669:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d180. 00000100:00000010:3.0:1790701612.175749:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce118f6c0. 00000100:00000010:2.0:1790701612.181062:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce118f780. 00000001:02000400:0.0F:1790701615.497811:0:129882:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 550: DD-off: keep route up when gw reachable via another local NI ========================================================== 13:06:53 (1790701613) 00000100:00000010:2.0:1790701616.012010:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671f80. 02000000:00000010:2.0:1790701616.012043:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4200. 00000100:00000010:2.0:1790701616.012086:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671180. 02000000:00000010:2.0:1790701616.012092:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4900. 00000100:00000010:2.0:1790701616.012100:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670700. 02000000:00000010:2.0:1790701616.012104:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4e00. 02000000:00000010:3.0:1790701616.017003:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bd200. 00000400:00000200:3.0:1790701616.017029:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701616.017035:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318. 00000400:00000010:3.0:1790701616.017045:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060770. 00000100:00000200:3.0:1790701616.017056:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432192, portal 25 00000400:00000010:3.0:1790701616.017063:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060ee0. 00000100:00000200:3.0:1790701616.017067:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537432192, offset 0 00000400:00000200:3.0:1790701616.017078:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701616.017088:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.017098:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701616.017103:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701616.017108:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701616.017112:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701616.017115:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701616.017119:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [94:94:256:4294967295] 192.168.204.157@tcp LPNI seq info [94:94:8:4294967295] 00000400:00000200:3.0:1790701616.017132:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.017144:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701616.017151:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701616.017158:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441400. 00000800:00000200:3.0:1790701616.017166:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701616.017175:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701616.017180:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701616.017206:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701616.017212:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d2a0. 00000100:00000010:3.0:1790701616.017229:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d000. 00000800:00000200:2.0:1790701616.017460:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701616.017467:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441400. 00000400:00000200:2.0:1790701616.017475:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701616.017480:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701616.017488:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060ee0 00000400:00000010:2.0:1790701616.017490:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060ee0. 00000100:00000200:2.0:1790701616.017499:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca671f80 x1877686537432192/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701616.017700:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bd400. 00000400:00000200:3.0:1790701616.017705:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701616.017709:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8. 00000400:00000010:3.0:1790701616.017714:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701616.017718:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432448, portal 10 00000400:00000010:3.0:1790701616.017725:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060110. 00000100:00000200:3.0:1790701616.017727:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537432448, offset 0 00000400:00000200:3.0:1790701616.017733:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701616.017737:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.017741:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701616.017743:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701616.017757:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701616.017760:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701616.017762:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701616.017765:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [95:95:256:4294967295] 192.168.204.157@tcp LPNI seq info [95:95:8:4294967295] 00000400:00000200:3.0:1790701616.017769:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.017776:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701616.017781:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701616.017786:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441e00. 00000800:00000200:3.0:1790701616.017791:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701616.017805:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701616.017808:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701616.017863:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701616.017893:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc63a4600. 00000400:00000200:3.0:1790701616.017897:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701616.017900:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b58. 00000400:00000010:3.0:1790701616.017903:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0607f8. 00000100:00000200:3.0:1790701616.017907:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432320, portal 10 00000400:00000010:3.0:1790701616.017912:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701616.017914:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537432320, offset 0 00000400:00000200:3.0:1790701616.017919:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701616.017923:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.017926:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701616.017929:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701616.017931:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701616.017934:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701616.017937:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701616.017939:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [96:96:255:4294967295] 192.168.204.157@tcp LPNI seq info [96:96:7:4294967295] 00000400:00000200:3.0:1790701616.017943:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.017949:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701616.017953:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701616.017957:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441d00. 00000800:00000200:3.0:1790701616.017961:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701616.017965:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701616.017967:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701616.017972:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701616.018077:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701616.018080:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441e00. 00000400:00000200:2.0:1790701616.018084:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701616.018088:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701616.018092:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060110 00000400:00000010:2.0:1790701616.018093:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060110. 00000100:00000200:2.0:1790701616.018100:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670700 x1877686537432448/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701616.018150:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701616.018153:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441d00. 00000400:00000200:2.0:1790701616.018155:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701616.018158:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701616.018161:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:2.0:1790701616.018162:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:2.0:1790701616.018167:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca671180 x1877686537432320/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701616.019665:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701616.019685:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701616.019696:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701616.019702:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701616.019708:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701616.019712:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102c80 00000400:00000200:2.0:1790701616.019718:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701616.019724:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x62d [1] + 224 00000400:00000200:2.0:1790701616.019744:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701616.019762:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701616.019776:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca671f80 x1877686537432192/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701616.019931:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318. 00000400:00000200:3.0:1790701616.019936:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060770 00000400:00000010:3.0:1790701616.019939:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060770. 00000100:00000200:3.0:1790701616.019945:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca671f80 x1877686537432192/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701616.019964:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca671f80 x1877686537432192/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701616.019995:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bd200. 02000000:00000010:3.0:1790701616.020003:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4200. 00000100:00000010:3.0:1790701616.020008:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671f80. 00000400:00000200:3.0:1790701616.023638:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023651:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701616.023660:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023666:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023672:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023677:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102d80 00000400:00000200:3.0:1790701616.023684:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023691:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x63d [1] + 224 00000400:00000200:3.0:1790701616.023706:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701616.023711:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701616.023730:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670700 x1877686537432448/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701616.023780:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023784:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701616.023789:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023791:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023794:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023797:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102d00 00000400:00000200:3.0:1790701616.023800:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701616.023804:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x64d [1] + 224 00000400:00000200:3.0:1790701616.023812:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701616.023815:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701616.024345:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca671180 x1877686537432320/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701616.024407:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b58. 00000400:00000200:3.0:1790701616.024412:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0607f8 00000400:00000010:3.0:1790701616.024416:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0607f8. 00000100:00000200:3.0:1790701616.024422:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca671180 x1877686537432320/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701616.024431:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca671180 x1877686537432320/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701616.024589:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8. 00000400:00000200:3.0:1790701616.024593:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:3.0:1790701616.024595:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701616.024600:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670700 x1877686537432448/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701616.024609:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670700 x1877686537432448/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701632 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701616.024628:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc63a4600. 02000000:00000010:3.0:1790701616.024634:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4900. 00000100:00000010:3.0:1790701616.024641:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671180. 02000000:00000010:3.0:1790701616.024646:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bd400. 02000000:00000010:3.0:1790701616.024649:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4e00. 00000100:00000010:3.0:1790701616.024651:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670700. 00000100:00000010:3.0:1790701617.355925:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d000. 00000100:00000010:2.0:1790701617.356109:0:120878:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d2a0. 00000001:02000400:2.0:1790701617.676387:0:130022:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_550 Need at least 2 local interfaces for multi-rail 00000001:02000400:3.0:1790701619.577313:0:130176:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 600: lnet_legacy2yaml: basic ip2nets conversion ========================================================== 13:06:58 (1790701618) 00000100:00000010:2.0:1790701621.133803:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671880. 02000000:00000010:2.0:1790701621.133851:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4500. 00000100:00000010:2.0:1790701621.133887:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca673800. 02000000:00000010:2.0:1790701621.133892:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4700. 00000100:00000010:2.0:1790701621.133899:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670380. 02000000:00000010:2.0:1790701621.133903:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4300. 02000000:00000010:2.0:1790701621.133941:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7ace600. 00000400:00000200:2.0:1790701621.133952:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701621.133956:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2058. 00000400:00000010:2.0:1790701621.133964:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a198. 00000100:00000200:2.0:1790701621.133972:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432576, portal 25 00000400:00000010:2.0:1790701621.133979:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701621.133982:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537432576, offset 0 00000400:00000200:2.0:1790701621.133991:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701621.133998:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701621.134006:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701621.134012:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701621.134030:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701621.134034:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701621.134037:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701621.134041:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [97:97:256:4294967295] 192.168.204.157@tcp LPNI seq info [97:97:8:4294967295] 00000400:00000200:2.0:1790701621.134046:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701621.134056:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701621.134062:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701621.134067:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4600. 00000800:00000200:2.0:1790701621.134074:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701621.134082:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701621.134086:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701621.134100:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:2.0:1790701621.134105:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc9edb600. 00000100:00000010:2.0:1790701621.134115:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc9edb0c0. 02000000:00000010:2.0:1790701621.134126:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7ace000. 00000400:00000200:2.0:1790701621.134130:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701621.134132:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b24d0. 00000400:00000010:2.0:1790701621.134137:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a2a8. 00000100:00000200:2.0:1790701621.134140:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432832, portal 10 00000400:00000010:2.0:1790701621.134146:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a990. 00000100:00000200:2.0:1790701621.134148:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537432832, offset 0 00000400:00000200:2.0:1790701621.134932:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701621.134939:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701621.134945:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701621.134949:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701621.134953:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701621.134955:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701621.134958:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701621.134961:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [98:98:255:4294967295] 192.168.204.157@tcp LPNI seq info [98:98:7:4294967295] 00000400:00000200:2.0:1790701621.134966:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701621.134972:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701621.134988:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701621.134992:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4100. 00000800:00000200:2.0:1790701621.134998:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701621.135004:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701621.135007:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701621.135014:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:2.0:1790701621.135037:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acf800. 00000400:00000200:2.0:1790701621.135041:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701621.135044:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2210. 00000400:00000010:2.0:1790701621.135048:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a5d8. 00000100:00000200:2.0:1790701621.135051:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432704, portal 10 00000400:00000010:2.0:1790701621.135057:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701621.135059:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537432704, offset 0 00000400:00000200:2.0:1790701621.135064:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701621.135067:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701621.135071:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701621.135073:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701621.135076:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701621.135078:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701621.135080:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701621.135083:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [99:99:254:4294967295] 192.168.204.157@tcp LPNI seq info [99:99:6:4294967295] 00000400:00000200:2.0:1790701621.135087:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701621.135092:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701621.135096:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701621.135100:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4000. 00000800:00000200:2.0:1790701621.135104:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701621.135109:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701621.135111:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701621.135116:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701621.136804:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701621.137008:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4600. 00000400:00000200:2.0:1790701621.137016:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701621.137025:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701621.137032:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a330 00000400:00000010:2.0:1790701621.137035:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701621.137048:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca671880 x1877686537432576/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701621.137081:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701621.137084:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4100. 00000400:00000200:2.0:1790701621.137087:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701621.137090:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701621.137092:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a990 00000400:00000010:2.0:1790701621.137094:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a990. 00000100:00000200:2.0:1790701621.137100:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670380 x1877686537432832/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701621.137645:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701621.137658:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4000. 00000400:00000200:2.0:1790701621.137662:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701621.137666:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701621.137670:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a220 00000400:00000010:2.0:1790701621.137673:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701621.137680:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca673800 x1877686537432704/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701621.146373:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146389:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701621.146399:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146405:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146412:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146416:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102e00 00000400:00000200:3.0:1790701621.146428:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146434:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x65d [1] + 224 00000400:00000200:3.0:1790701621.146447:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701621.146452:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701621.146467:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca671880 x1877686537432576/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701621.146518:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701621.146521:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2058. 00000400:00000200:3.0:1790701621.146523:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701621.146529:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a198 00000400:00000200:3.0:1790701621.146533:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701621.146534:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a198. 00000400:00000200:3.0:1790701621.146539:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146543:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000200:2.0:1790701621.146547:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca671880 x1877686537432576/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701621.146548:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102f00 00000400:00000200:3.0:1790701621.146553:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146556:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x66d [1] + 224 00000400:00000200:3.0:1790701621.146562:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701621.146565:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701621.146571:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670380 x1877686537432832/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701621.146585:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146588:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701621.146592:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146594:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146597:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146599:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102e80 00000400:00000200:3.0:1790701621.146602:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701621.146606:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x67d [1] + 224 00000400:00000200:3.0:1790701621.146613:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701621.146616:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701621.146622:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca673800 x1877686537432704/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701621.146683:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca671880 x1877686537432576/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701621.146714:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b24d0. 00000400:00000200:2.0:1790701621.146723:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a2a8 00000400:00000010:2.0:1790701621.146725:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a2a8. 00000100:00000200:2.0:1790701621.146731:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670380 x1877686537432832/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701621.146740:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670380 x1877686537432832/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701621.146764:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7ace600. 02000000:00000010:2.0:1790701621.146771:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4500. 00000100:00000010:2.0:1790701621.146776:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671880. 02000000:00000010:2.0:1790701621.146781:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7ace000. 02000000:00000010:2.0:1790701621.146784:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4300. 00000100:00000010:2.0:1790701621.146786:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670380. 00000400:00000010:2.0:1790701621.146909:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2210. 00000400:00000200:2.0:1790701621.146913:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a5d8 00000400:00000010:2.0:1790701621.146915:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a5d8. 00000100:00000200:2.0:1790701621.146921:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca673800 x1877686537432704/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701621.146930:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca673800 x1877686537432704/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701637 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701621.146948:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acf800. 02000000:00000010:2.0:1790701621.146951:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4700. 00000100:00000010:2.0:1790701621.146954:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca673800. 00000100:00000010:3.0:1790701622.422772:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc9edb0c0. 00000100:00000010:2.0:1790701622.423132:0:120878:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc9edb600. 00000100:00000010:1.0F:1790701626.258137:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701626.258178:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5000. 00000100:00000010:1.0:1790701626.258220:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701626.258230:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5e00. 00000100:00000010:1.0:1790701626.258238:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701626.258241:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5500. 02000000:00000010:1.0:1790701626.258281:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc37f1000. 00000400:00000200:1.0:1790701626.258292:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701626.258296:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701626.258302:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2ee0. 00000100:00000200:1.0:1790701626.258311:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537432960, portal 25 00000400:00000010:1.0:1790701626.258317:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2dd0. 00000100:00000200:1.0:1790701626.258321:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537432960, offset 0 00000400:00000200:1.0:1790701626.258332:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701626.258339:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701626.258347:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701626.258352:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701626.258357:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701626.258360:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701626.258363:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701626.258367:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [100:100:256:4294967295] 192.168.204.157@tcp LPNI seq info [100:100:8:4294967295] 00000400:00000200:1.0:1790701626.258371:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701626.258382:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701626.258388:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701626.258392:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5400. 00000800:00000200:1.0:1790701626.258399:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701626.258407:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701626.258411:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701626.258426:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701626.258436:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc913d900. 00000100:00000010:1.0:1790701626.258451:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc913dcc0. 02000000:00000010:1.0:1790701626.258463:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc37f1600. 00000400:00000200:1.0:1790701626.258474:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701626.258476:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701626.258479:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701626.258482:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433216, portal 10 00000400:00000010:1.0:1790701626.258488:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b23b8. 00000100:00000200:1.0:1790701626.258490:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537433216, offset 0 00000400:00000200:1.0:1790701626.258495:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701626.258498:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701626.258501:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701626.258504:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701626.258507:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701626.258509:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701626.258511:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701626.258514:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [101:101:255:4294967295] 192.168.204.157@tcp LPNI seq info [101:101:7:4294967295] 00000400:00000200:1.0:1790701626.258518:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701626.258524:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701626.258529:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701626.258532:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5d00. 00000800:00000200:1.0:1790701626.258536:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701626.258540:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701626.258542:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701626.258558:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701626.258578:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccc4d3400. 00000400:00000200:1.0:1790701626.258581:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701626.258584:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701626.258587:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2880. 00000100:00000200:1.0:1790701626.258590:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433088, portal 10 00000400:00000010:1.0:1790701626.258599:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2000. 00000100:00000200:1.0:1790701626.258602:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537433088, offset 0 00000400:00000200:1.0:1790701626.258606:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701626.258609:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701626.258618:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701626.258621:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701626.258624:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701626.258626:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701626.258629:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701626.258631:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [102:102:254:4294967295] 192.168.204.157@tcp LPNI seq info [102:102:6:4294967295] 00000400:00000200:1.0:1790701626.258636:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701626.258641:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701626.258646:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701626.258649:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5800. 00000800:00000200:1.0:1790701626.258654:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701626.258659:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701626.258661:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701626.258666:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701626.259037:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701626.259043:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5400. 00000400:00000200:2.0:1790701626.259051:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701626.259058:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701626.259064:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2dd0 00000400:00000010:2.0:1790701626.259066:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2dd0. 00000100:00000200:2.0:1790701626.259089:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537432960/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701642 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701626.259126:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701626.259128:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5d00. 00000400:00000200:2.0:1790701626.259132:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701626.259136:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701626.259139:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b23b8 00000400:00000010:2.0:1790701626.259140:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b23b8. 00000100:00000200:2.0:1790701626.259145:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537433216/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701626.259215:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701626.259218:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5800. 00000400:00000200:2.0:1790701626.259221:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701626.259225:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701626.259228:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2000 00000400:00000010:2.0:1790701626.259229:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2000. 00000100:00000200:2.0:1790701626.259234:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537433088/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701626.272197:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272209:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701626.272219:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272225:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272231:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272235:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103080 00000400:00000200:3.0:1790701626.272240:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272247:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x695 [1] + 224 00000400:00000200:3.0:1790701626.272259:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701626.272264:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701626.272281:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537433216/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701626.272351:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272355:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701626.272360:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272363:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272366:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272368:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103000 00000400:00000200:3.0:1790701626.272372:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272375:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x69d [1] + 224 00000400:00000200:3.0:1790701626.272382:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701626.272385:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701626.272392:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537433088/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701626.272406:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272408:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701626.272412:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272423:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272426:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272429:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102f80 00000400:00000200:3.0:1790701626.272432:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701626.272436:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x68d [1] + 224 00000400:00000200:3.0:1790701626.272443:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701626.272447:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701626.272453:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537432960/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701626.272701:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701626.272706:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2880 00000400:00000010:1.0:1790701626.272709:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2880. 00000100:00000200:1.0:1790701626.272715:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537433088/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701626.272725:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537433088/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701626.272757:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701626.272760:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2ee0 00000400:00000010:1.0:1790701626.272762:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2ee0. 00000100:00000200:1.0:1790701626.272767:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537432960/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701626.272775:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537432960/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701626.272784:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701626.272793:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701626.272794:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701626.272799:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537433216/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701626.272808:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537433216/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701642 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701626.272859:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccc4d3400. 02000000:00000010:1.0:1790701626.272866:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5e00. 00000100:00000010:1.0:1790701626.272874:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701626.272879:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc37f1000. 02000000:00000010:1.0:1790701626.272883:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5000. 00000100:00000010:1.0:1790701626.272885:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701626.272890:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc37f1600. 02000000:00000010:1.0:1790701626.272892:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5500. 00000100:00000010:1.0:1790701626.272894:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000100:00000010:2.0:1790701627.596786:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc913d900. 00000100:00000010:3.0:1790701627.596892:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc913dcc0. 00000001:02000400:2.0:1790701628.661006:0:130757:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 601: lnet_legacy2yaml: basic routes conversion ========================================================== 13:07:07 (1790701627) 00000100:00000010:3.0:1790701631.373561:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d180. 02000000:00000010:3.0:1790701631.373583:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441e00. 00000100:00000010:3.0:1790701631.373616:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d500. 02000000:00000010:3.0:1790701631.373621:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441400. 00000100:00000010:3.0:1790701631.373627:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c000. 02000000:00000010:3.0:1790701631.373631:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441d00. 02000000:00000010:3.0:1790701631.373668:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd8b64400. 00000400:00000200:3.0:1790701631.373679:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701631.373684:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8. 00000400:00000010:3.0:1790701631.373691:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701631.373698:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433344, portal 25 00000400:00000010:3.0:1790701631.373704:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0607f8. 00000100:00000200:3.0:1790701631.373706:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537433344, offset 0 00000400:00000200:3.0:1790701631.373716:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701631.373724:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.373732:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701631.373737:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701631.373742:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701631.373746:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701631.373749:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701631.373753:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [103:103:256:4294967295] 192.168.204.157@tcp LPNI seq info [103:103:8:4294967295] 00000400:00000200:3.0:1790701631.373768:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.373779:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701631.373786:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701631.373790:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441900. 00000800:00000200:3.0:1790701631.373797:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701631.373806:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701631.373810:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701631.373844:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701631.373850:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4d80. 00000100:00000010:3.0:1790701631.373860:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4060. 02000000:00000010:3.0:1790701631.373873:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd8b5f600. 00000400:00000200:3.0:1790701631.373877:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701631.373880:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b58. 00000400:00000010:3.0:1790701631.373883:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060770. 00000100:00000200:3.0:1790701631.373886:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433600, portal 10 00000400:00000010:3.0:1790701631.373892:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701631.373894:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537433600, offset 0 00000400:00000200:3.0:1790701631.373898:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701631.373902:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.373906:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701631.373909:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701631.373911:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701631.373913:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701631.373915:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701631.373918:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [104:104:255:4294967295] 192.168.204.157@tcp LPNI seq info [104:104:7:4294967295] 00000400:00000200:3.0:1790701631.373921:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.373927:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701631.373932:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701631.373935:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441000. 00000800:00000200:3.0:1790701631.373938:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701631.373948:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701631.373951:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701631.373958:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701631.373979:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc6699200. 00000400:00000200:3.0:1790701631.373982:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701631.373985:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318. 00000400:00000010:3.0:1790701631.373988:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701631.373991:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433472, portal 10 00000400:00000010:3.0:1790701631.373995:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060110. 00000100:00000200:3.0:1790701631.373998:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537433472, offset 0 00000400:00000200:3.0:1790701631.374002:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701631.374006:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.374009:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701631.374012:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701631.374014:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701631.374017:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701631.374019:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701631.374022:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [105:105:254:4294967295] 192.168.204.157@tcp LPNI seq info [105:105:6:4294967295] 00000400:00000200:3.0:1790701631.374026:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.374032:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701631.374036:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701631.374039:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441f00. 00000800:00000200:3.0:1790701631.374043:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701631.374047:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701631.374049:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701631.374054:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701631.374101:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701631.374105:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441900. 00000400:00000200:3.0:1790701631.374109:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701631.374113:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701631.374119:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0607f8 00000400:00000010:3.0:1790701631.374127:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0607f8. 00000100:00000200:3.0:1790701631.374138:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d180 x1877686537433344/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701631.374160:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701631.374162:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441000. 00000400:00000200:3.0:1790701631.374165:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701631.374168:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701631.374171:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060440 00000400:00000010:3.0:1790701631.374173:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701631.374177:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074c000 x1877686537433600/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701631.374250:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701631.374253:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441f00. 00000400:00000200:3.0:1790701631.374255:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701631.374259:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701631.374261:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060110 00000400:00000010:3.0:1790701631.374263:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060110. 00000100:00000200:3.0:1790701631.374268:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d500 x1877686537433472/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701631.380842:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380854:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701631.380864:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380870:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380876:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380880:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103200 00000400:00000200:3.0:1790701631.380886:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380892:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x6b5 [1] + 224 00000400:00000200:3.0:1790701631.380904:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701631.380909:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701631.380932:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074c000 x1877686537433600/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701631.380986:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380990:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701631.380994:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.380997:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.381000:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.381003:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103180 00000400:00000200:3.0:1790701631.381006:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.381010:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x6c5 [1] + 224 00000400:00000200:3.0:1790701631.381019:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701631.381022:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701631.381029:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d500 x1877686537433472/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701631.381063:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318. 00000400:00000200:3.0:1790701631.381068:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:3.0:1790701631.381070:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701631.381077:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d500 x1877686537433472/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701631.381085:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d500 x1877686537433472/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701631.381126:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b58. 00000400:00000200:3.0:1790701631.381129:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060770 00000400:00000010:3.0:1790701631.381131:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060770. 00000100:00000200:3.0:1790701631.381135:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074c000 x1877686537433600/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701631.381143:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074c000 x1877686537433600/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701631.381161:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc6699200. 02000000:00000010:3.0:1790701631.381174:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441400. 00000100:00000010:3.0:1790701631.381179:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d500. 02000000:00000010:3.0:1790701631.381183:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd8b5f600. 02000000:00000010:3.0:1790701631.381188:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441d00. 00000100:00000010:3.0:1790701631.381190:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c000. 00000400:00000200:3.0:1790701631.386507:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.386517:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701631.386526:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.386531:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.386537:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.386541:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103100 00000400:00000200:3.0:1790701631.386546:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701631.386553:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x6a5 [1] + 224 00000400:00000200:3.0:1790701631.386567:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701631.386571:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701631.386587:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d180 x1877686537433344/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701631.386632:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8. 00000400:00000200:3.0:1790701631.386636:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:3.0:1790701631.386638:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701631.386643:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d180 x1877686537433344/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701631.386652:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d180 x1877686537433344/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701647 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701631.386686:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd8b64400. 02000000:00000010:3.0:1790701631.386694:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441e00. 00000100:00000010:3.0:1790701631.386698:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d180. 00000100:00000010:2.0:1790701632.651931:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4d80. 00000100:00000010:3.0:1790701632.652209:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4060. 00000100:00000010:2.0:1790701636.497473:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca673800. 02000000:00000010:2.0:1790701636.497494:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4000. 00000100:00000010:2.0:1790701636.497529:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670380. 02000000:00000010:2.0:1790701636.497535:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4500. 00000100:00000010:2.0:1790701636.497541:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671880. 02000000:00000010:2.0:1790701636.497545:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4100. 02000000:00000010:3.0:1790701636.499085:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319b000. 00000400:00000200:3.0:1790701636.499101:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701636.499118:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8. 00000400:00000010:3.0:1790701636.499125:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701636.499133:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433728, portal 25 00000400:00000010:3.0:1790701636.499140:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060770. 00000100:00000200:3.0:1790701636.499143:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537433728, offset 0 00000400:00000200:3.0:1790701636.499152:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701636.499160:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701636.499169:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701636.499174:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701636.499179:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701636.499183:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701636.499186:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701636.499189:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [106:106:256:4294967295] 192.168.204.157@tcp LPNI seq info [106:106:8:4294967295] 00000400:00000200:3.0:1790701636.499194:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701636.499202:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701636.499210:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701636.499215:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441f00. 00000800:00000200:3.0:1790701636.499222:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701636.499230:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701636.499235:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701636.499331:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701636.499338:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e42a0. 00000100:00000010:3.0:1790701636.499359:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e44e0. 02000000:00000010:3.0:1790701636.499373:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319aa00. 00000400:00000200:3.0:1790701636.499377:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701636.499379:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b58. 00000400:00000010:3.0:1790701636.499383:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701636.499386:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433984, portal 10 00000400:00000010:3.0:1790701636.499391:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060110. 00000100:00000200:3.0:1790701636.499394:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537433984, offset 0 00000400:00000200:3.0:1790701636.499398:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701636.499407:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701636.499410:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701636.499413:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701636.499415:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701636.499418:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701636.499420:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701636.499423:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [107:107:255:4294967295] 192.168.204.157@tcp LPNI seq info [107:107:7:4294967295] 00000400:00000200:3.0:1790701636.499427:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701636.499433:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701636.499437:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701636.499441:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441100. 00000800:00000200:3.0:1790701636.499444:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701636.499449:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701636.499451:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701636.499457:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701636.499479:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319ae00. 00000400:00000200:3.0:1790701636.499483:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701636.499485:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318. 00000400:00000010:3.0:1790701636.499488:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701636.499491:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537433856, portal 10 00000400:00000010:3.0:1790701636.499494:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0607f8. 00000100:00000200:3.0:1790701636.499497:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537433856, offset 0 00000400:00000200:3.0:1790701636.499501:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701636.499505:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701636.499508:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701636.499511:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701636.499513:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701636.499514:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701636.499517:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701636.499519:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [108:108:254:4294967295] 192.168.204.157@tcp LPNI seq info [108:108:6:4294967295] 00000400:00000200:3.0:1790701636.499523:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701636.499535:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701636.499540:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701636.499544:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441a00. 00000800:00000200:3.0:1790701636.499547:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701636.499552:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701636.499555:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701636.499560:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701636.499682:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701636.499686:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441f00. 00000400:00000200:2.0:1790701636.499692:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701636.499697:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701636.499707:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060770 00000400:00000010:2.0:1790701636.499709:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060770. 00000100:00000200:2.0:1790701636.499720:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca673800 x1877686537433728/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701636.499743:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701636.499746:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441100. 00000400:00000200:2.0:1790701636.499749:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701636.499752:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701636.499754:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060110 00000400:00000010:2.0:1790701636.499756:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060110. 00000100:00000200:2.0:1790701636.499761:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca671880 x1877686537433984/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701636.500183:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701636.500186:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441a00. 00000400:00000200:2.0:1790701636.500190:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701636.500194:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701636.500196:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0607f8 00000400:00000010:2.0:1790701636.500198:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0607f8. 00000100:00000200:2.0:1790701636.500204:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670380 x1877686537433856/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701636.518674:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518685:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701636.518693:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518698:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518705:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518709:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103380 00000400:00000200:2.0:1790701636.518715:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518721:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x6e5 [1] + 224 00000400:00000200:2.0:1790701636.518732:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701636.518737:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701636.518755:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca671880 x1877686537433984/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701636.518791:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518795:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701636.518799:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518801:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518804:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518807:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103280 00000400:00000200:2.0:1790701636.518810:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.518838:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x6d5 [1] + 224 00000400:00000200:2.0:1790701636.518846:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701636.518849:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701636.518854:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca673800 x1877686537433728/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701636.518868:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.519331:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701636.519339:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.519343:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.519357:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.519361:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103300 00000400:00000200:2.0:1790701636.519365:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701636.519369:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x6f5 [1] + 224 00000400:00000200:2.0:1790701636.519382:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701636.519386:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701636.519392:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670380 x1877686537433856/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701636.521387:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318. 00000400:00000200:3.0:1790701636.521398:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060440 00000400:00000010:3.0:1790701636.521400:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701636.521415:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670380 x1877686537433856/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701636.521427:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670380 x1877686537433856/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701636.521462:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8. 00000400:00000200:3.0:1790701636.521465:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:3.0:1790701636.521466:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701636.521471:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca673800 x1877686537433728/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701636.521478:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca673800 x1877686537433728/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701636.521489:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b58. 00000400:00000200:3.0:1790701636.521491:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:3.0:1790701636.521493:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701636.521506:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca671880 x1877686537433984/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701636.521514:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca671880 x1877686537433984/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701652 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701636.521531:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319ae00. 02000000:00000010:3.0:1790701636.521543:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4500. 00000100:00000010:3.0:1790701636.521548:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670380. 02000000:00000010:3.0:1790701636.521552:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319b000. 02000000:00000010:3.0:1790701636.521555:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4000. 00000100:00000010:3.0:1790701636.521557:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca673800. 02000000:00000010:3.0:1790701636.521562:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319aa00. 02000000:00000010:3.0:1790701636.521564:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4100. 00000100:00000010:3.0:1790701636.521566:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671880. 00000100:00000010:2.0:1790701637.835965:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e42a0. 00000100:00000010:3.0:1790701637.836016:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e44e0. 00000001:02000400:2.0:1790701638.106704:0:131338:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 602: lnet_legacy2yaml: routes with priority ========================================================== 13:07:16 (1790701636) 00000100:00000010:2.0:1790701641.625322:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670a80. 02000000:00000010:2.0:1790701641.625350:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4600. 00000100:00000010:2.0:1790701641.625381:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671500. 02000000:00000010:2.0:1790701641.625385:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4f00. 00000100:00000010:2.0:1790701641.625393:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca673b80. 02000000:00000010:2.0:1790701641.625397:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4400. 02000000:00000010:3.0:1790701641.632417:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929a00. 00000400:00000200:3.0:1790701641.632436:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701641.632440:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b58. 00000400:00000010:3.0:1790701641.632449:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701641.632459:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434112, portal 25 00000400:00000010:3.0:1790701641.632465:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701641.632468:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537434112, offset 0 00000400:00000200:3.0:1790701641.632477:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701641.632485:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.632492:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701641.632497:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701641.632503:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701641.632508:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701641.632510:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701641.632514:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [109:109:256:4294967295] 192.168.204.157@tcp LPNI seq info [109:109:8:4294967295] 00000400:00000200:3.0:1790701641.632519:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.632531:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701641.632539:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701641.632546:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441000. 00000800:00000200:3.0:1790701641.632554:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701641.632562:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701641.632567:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701641.632594:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701641.632599:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4480. 00000100:00000010:3.0:1790701641.632613:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4900. 02000000:00000010:3.0:1790701641.632626:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928600. 00000400:00000200:3.0:1790701641.632630:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701641.632632:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8. 00000400:00000010:3.0:1790701641.632636:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701641.632639:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434368, portal 10 00000400:00000010:3.0:1790701641.632645:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060ee0. 00000100:00000200:3.0:1790701641.632647:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537434368, offset 0 00000400:00000200:3.0:1790701641.632651:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701641.632655:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.632658:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701641.632661:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701641.632663:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701641.632666:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701641.632668:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701641.632671:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [110:110:255:4294967295] 192.168.204.157@tcp LPNI seq info [110:110:7:4294967295] 00000400:00000200:3.0:1790701641.632674:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.632680:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701641.632684:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701641.632688:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441900. 00000800:00000200:3.0:1790701641.632691:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701641.632695:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701641.632697:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701641.632703:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701641.632722:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929200. 00000400:00000200:3.0:1790701641.632726:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701641.632728:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318. 00000400:00000010:3.0:1790701641.632731:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701641.632734:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434240, portal 10 00000400:00000010:3.0:1790701641.632746:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060880. 00000100:00000200:3.0:1790701641.632748:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537434240, offset 0 00000400:00000200:3.0:1790701641.632767:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701641.632771:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.632774:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701641.632777:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701641.632779:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701641.632781:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701641.632783:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701641.632786:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [111:111:254:4294967295] 192.168.204.157@tcp LPNI seq info [111:111:6:4294967295] 00000400:00000200:3.0:1790701641.632790:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.632796:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701641.632800:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701641.632803:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441c00. 00000800:00000200:3.0:1790701641.632807:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701641.632814:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701641.632832:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701641.632840:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701641.632909:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701641.632913:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441000. 00000400:00000200:3.0:1790701641.632918:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701641.632923:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701641.632928:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:3.0:1790701641.632931:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701641.633225:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670a80 x1877686537434112/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701641.633264:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701641.633266:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441900. 00000400:00000200:3.0:1790701641.633269:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701641.633272:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701641.633275:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060ee0 00000400:00000010:3.0:1790701641.633282:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060ee0. 00000100:00000200:3.0:1790701641.633288:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca673b80 x1877686537434368/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701641.633366:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701641.633369:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441c00. 00000400:00000200:3.0:1790701641.633372:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701641.633375:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701641.633378:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060880 00000400:00000010:3.0:1790701641.633379:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060880. 00000100:00000200:3.0:1790701641.633383:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca671500 x1877686537434240/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701641.640435:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.640446:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701641.640456:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.640461:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.640467:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.640470:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103400 00000400:00000200:3.0:1790701641.640475:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.640481:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x705 [1] + 224 00000400:00000200:3.0:1790701641.640495:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701641.640500:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701641.640516:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670a80 x1877686537434112/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701641.640557:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b58. 00000400:00000200:3.0:1790701641.640562:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:3.0:1790701641.640564:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701641.640570:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670a80 x1877686537434112/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701641.640579:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670a80 x1877686537434112/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701641.640663:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929a00. 02000000:00000010:3.0:1790701641.640669:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4600. 00000100:00000010:3.0:1790701641.640674:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670a80. 00000400:00000200:3.0:1790701641.649418:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649431:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701641.649440:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649446:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649453:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649457:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103500 00000400:00000200:3.0:1790701641.649463:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649469:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x715 [1] + 224 00000400:00000200:3.0:1790701641.649481:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701641.649486:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701641.649504:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca673b80 x1877686537434368/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701641.649531:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649534:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701641.649537:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649540:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649542:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649545:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103480 00000400:00000200:3.0:1790701641.649548:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701641.649551:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x725 [1] + 224 00000400:00000200:3.0:1790701641.649559:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701641.649605:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701641.649612:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca671500 x1877686537434240/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701641.649645:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318. 00000400:00000200:3.0:1790701641.649664:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060aa0 00000400:00000010:3.0:1790701641.649666:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701641.649672:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca671500 x1877686537434240/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701641.649681:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca671500 x1877686537434240/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701641.649717:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8. 00000400:00000200:3.0:1790701641.649720:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060440 00000400:00000010:3.0:1790701641.649722:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701641.649726:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca673b80 x1877686537434368/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701641.649734:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca673b80 x1877686537434368/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701657 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701641.649751:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929200. 02000000:00000010:3.0:1790701641.649757:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4f00. 00000100:00000010:3.0:1790701641.649763:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671500. 02000000:00000010:3.0:1790701641.649767:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928600. 02000000:00000010:3.0:1790701641.649770:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4400. 00000100:00000010:3.0:1790701641.649772:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca673b80. 00000100:00000010:2.0:1790701642.955961:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4480. 00000100:00000010:3.0:1790701642.959693:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4900. 00000100:00000010:1.0:1790701646.732567:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701646.732592:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5d00. 00000100:00000010:1.0:1790701646.732625:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701646.732630:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701646.732638:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701646.732642:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5400. 02000000:00000010:1.0:1790701646.732681:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce12ae600. 00000400:00000200:1.0:1790701646.732691:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701646.732696:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701646.732703:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701646.732711:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434496, portal 25 00000400:00000010:1.0:1790701646.732717:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2ee0. 00000100:00000200:1.0:1790701646.732720:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537434496, offset 0 00000400:00000200:1.0:1790701646.732729:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701646.732736:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701646.732745:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701646.732763:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701646.732768:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701646.732772:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701646.732774:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701646.732778:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [112:112:256:4294967295] 192.168.204.157@tcp LPNI seq info [112:112:8:4294967295] 00000400:00000200:1.0:1790701646.732783:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701646.732793:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701646.732799:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701646.732803:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5200. 00000800:00000200:1.0:1790701646.732811:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701646.732843:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701646.732848:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701646.732862:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701646.732868:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd960. 00000100:00000010:1.0:1790701646.732878:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd1e0. 02000000:00000010:1.0:1790701646.732893:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccc48ba00. 00000400:00000200:1.0:1790701646.732897:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701646.732899:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701646.732902:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2880. 00000100:00000200:1.0:1790701646.732905:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434752, portal 10 00000400:00000010:1.0:1790701646.732911:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2220. 00000100:00000200:1.0:1790701646.732913:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537434752, offset 0 00000400:00000200:1.0:1790701646.732918:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701646.732922:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701646.732925:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701646.732928:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701646.732930:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701646.732932:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701646.732934:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701646.732936:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [113:113:255:4294967295] 192.168.204.157@tcp LPNI seq info [113:113:7:4294967295] 00000400:00000200:1.0:1790701646.732940:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701646.732952:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701646.732956:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701646.732959:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5f00. 00000800:00000200:1.0:1790701646.732963:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701646.732967:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701646.732969:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701646.732974:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701646.732992:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc9b4d000. 00000400:00000200:1.0:1790701646.732996:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701646.732998:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701646.733001:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701646.733003:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434624, portal 10 00000400:00000010:1.0:1790701646.733007:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701646.733009:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537434624, offset 0 00000400:00000200:1.0:1790701646.733013:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701646.733016:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701646.733019:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701646.733021:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701646.733023:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701646.733025:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701646.733027:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701646.733030:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [114:114:254:4294967295] 192.168.204.157@tcp LPNI seq info [114:114:6:4294967295] 00000400:00000200:1.0:1790701646.733033:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701646.733038:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701646.733042:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701646.733045:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5600. 00000800:00000200:1.0:1790701646.733048:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701646.733052:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701646.733054:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701646.733058:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701646.733172:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701646.733176:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5200. 00000400:00000200:2.0:1790701646.733182:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701646.733188:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701646.733194:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2ee0 00000400:00000010:2.0:1790701646.733196:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2ee0. 00000100:00000200:2.0:1790701646.733207:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537434496/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701646.733229:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701646.733231:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5f00. 00000400:00000200:2.0:1790701646.733234:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701646.733237:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701646.733240:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2220 00000400:00000010:2.0:1790701646.733241:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2220. 00000100:00000200:2.0:1790701646.733246:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537434752/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701646.733317:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701646.733320:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5600. 00000400:00000200:2.0:1790701646.733322:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701646.733326:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701646.733328:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2990 00000400:00000010:2.0:1790701646.733330:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2990. 00000100:00000200:2.0:1790701646.733334:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537434624/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701646.742638:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742662:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701646.742672:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742678:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742686:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742690:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103580 00000400:00000200:2.0:1790701646.742696:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742713:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x735 [1] + 224 00000400:00000200:2.0:1790701646.742727:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701646.742732:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701646.742749:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537434496/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701646.742786:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742790:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701646.742794:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742796:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742799:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742802:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103680 00000400:00000200:2.0:1790701646.742805:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.742809:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x73d [1] + 224 00000400:00000200:2.0:1790701646.742859:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701646.742863:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701646.742870:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537434752/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701646.743043:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701646.743049:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701646.743052:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701646.743060:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537434496/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701646.743070:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537434496/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701646.743099:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701646.743102:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2880 00000400:00000010:1.0:1790701646.743104:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2880. 00000100:00000200:1.0:1790701646.743108:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537434752/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701646.743116:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537434752/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701646.743137:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce12ae600. 02000000:00000010:1.0:1790701646.743145:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5d00. 00000100:00000010:1.0:1790701646.743164:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701646.743169:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccc48ba00. 02000000:00000010:1.0:1790701646.743173:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5400. 00000100:00000010:1.0:1790701646.743175:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000200:2.0:1790701646.757114:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.757127:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701646.757136:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.757142:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.757161:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.757166:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103600 00000400:00000200:2.0:1790701646.757171:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701646.757185:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x745 [1] + 224 00000400:00000200:2.0:1790701646.757201:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701646.757207:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701646.757223:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537434624/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701646.757354:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701646.757360:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b25d8 00000400:00000010:1.0:1790701646.757362:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701646.757370:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537434624/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701646.757380:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537434624/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701662 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701646.757421:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc9b4d000. 02000000:00000010:1.0:1790701646.757428:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701646.757433:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000001:02000400:3.0:1790701647.312068:0:131918:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 603: lnet_legacy2yaml: default hop flag ========================================================== 13:07:25 (1790701645) 00000100:00000010:2.0:1790701648.011909:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd960. 00000100:00000010:3.0:1790701648.011981:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd1e0. 00000100:00000010:3.0:1790701651.857735:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d180. 02000000:00000010:3.0:1790701651.857758:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441c00. 00000100:00000010:3.0:1790701651.857788:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c000. 02000000:00000010:3.0:1790701651.857791:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441e00. 00000100:00000010:3.0:1790701651.857797:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d500. 02000000:00000010:3.0:1790701651.857800:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441000. 02000000:00000010:3.0:1790701651.857857:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acc400. 00000400:00000200:3.0:1790701651.857890:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701651.857895:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8. 00000400:00000010:3.0:1790701651.857902:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701651.857908:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537434880, portal 25 00000400:00000010:3.0:1790701651.857914:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701651.857917:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537434880, offset 0 00000400:00000200:3.0:1790701651.857925:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701651.857933:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.857941:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701651.857945:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701651.857950:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701651.857954:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701651.857957:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701651.857960:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [115:115:256:4294967295] 192.168.204.157@tcp LPNI seq info [115:115:8:4294967295] 00000400:00000200:3.0:1790701651.857965:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.857974:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701651.857979:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701651.857983:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441200. 00000800:00000200:3.0:1790701651.857990:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701651.857998:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701651.858003:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701651.858015:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701651.858020:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e43c0. 00000100:00000010:3.0:1790701651.858029:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e48a0. 02000000:00000010:3.0:1790701651.858038:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acd200. 00000400:00000200:3.0:1790701651.858042:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701651.858044:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318. 00000400:00000010:3.0:1790701651.858059:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701651.858062:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435136, portal 10 00000400:00000010:3.0:1790701651.858068:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060880. 00000100:00000200:3.0:1790701651.858070:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537435136, offset 0 00000400:00000200:3.0:1790701651.858081:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701651.858085:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.858088:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701651.858091:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701651.858093:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701651.858095:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701651.858098:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701651.858101:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [116:116:255:4294967295] 192.168.204.157@tcp LPNI seq info [116:116:7:4294967295] 00000400:00000200:3.0:1790701651.858105:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.858111:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701651.858116:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701651.858120:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441300. 00000800:00000200:3.0:1790701651.858123:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701651.858127:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701651.858130:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701651.858136:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701651.858152:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acce00. 00000400:00000200:3.0:1790701651.858155:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701651.858158:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b58. 00000400:00000010:3.0:1790701651.858161:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060ee0. 00000100:00000200:3.0:1790701651.858163:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435008, portal 10 00000400:00000010:3.0:1790701651.858167:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701651.858170:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537435008, offset 0 00000400:00000200:3.0:1790701651.858174:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701651.858177:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.858180:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701651.858183:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701651.858185:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701651.858187:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701651.858189:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701651.858192:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [117:117:254:4294967295] 192.168.204.157@tcp LPNI seq info [117:117:6:4294967295] 00000400:00000200:3.0:1790701651.858201:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.858207:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701651.858212:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701651.858215:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441700. 00000800:00000200:3.0:1790701651.858219:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701651.858222:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701651.858224:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701651.858229:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701651.858276:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701651.858280:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441200. 00000400:00000200:3.0:1790701651.858286:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701651.858291:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701651.858296:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060aa0 00000400:00000010:3.0:1790701651.858298:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701651.858309:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d180 x1877686537434880/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701651.858331:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701651.858333:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441300. 00000400:00000200:3.0:1790701651.858336:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701651.858339:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701651.858341:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060880 00000400:00000010:3.0:1790701651.858343:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060880. 00000100:00000200:3.0:1790701651.858347:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d500 x1877686537435136/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701651.858414:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701651.858417:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441700. 00000400:00000200:3.0:1790701651.858420:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701651.858423:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701651.858425:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:3.0:1790701651.858427:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701651.858431:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074c000 x1877686537435008/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701651.863427:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863440:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701651.863450:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863455:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863461:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863465:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103800 00000400:00000200:3.0:1790701651.863471:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863477:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x75d [1] + 224 00000400:00000200:3.0:1790701651.863489:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701651.863494:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701651.863510:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d500 x1877686537435136/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701651.863543:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863547:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701651.863551:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863554:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863557:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863559:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103780 00000400:00000200:3.0:1790701651.863562:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.863565:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x76d [1] + 224 00000400:00000200:3.0:1790701651.863574:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701651.863577:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701651.863584:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074c000 x1877686537435008/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701651.863622:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b58. 00000400:00000200:3.0:1790701651.863627:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060ee0 00000400:00000010:3.0:1790701651.863630:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060ee0. 00000100:00000200:3.0:1790701651.863637:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074c000 x1877686537435008/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701651.863654:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074c000 x1877686537435008/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701651.863689:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318. 00000400:00000200:3.0:1790701651.863692:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:3.0:1790701651.863693:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701651.863697:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d500 x1877686537435136/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701651.863705:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d500 x1877686537435136/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701651.863721:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acce00. 02000000:00000010:3.0:1790701651.863726:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441e00. 00000100:00000010:3.0:1790701651.863731:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c000. 02000000:00000010:3.0:1790701651.863735:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acd200. 02000000:00000010:3.0:1790701651.863737:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441000. 00000100:00000010:3.0:1790701651.863740:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d500. 00000400:00000200:3.0:1790701651.864450:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.864457:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701651.864463:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.864467:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.864471:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.864474:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103700 00000400:00000200:3.0:1790701651.864478:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701651.864482:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x74d [1] + 224 00000400:00000200:3.0:1790701651.864491:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701651.864495:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701651.864506:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d180 x1877686537434880/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701651.864540:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8. 00000400:00000200:3.0:1790701651.864551:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060440 00000400:00000010:3.0:1790701651.864553:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701651.864558:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d180 x1877686537434880/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701651.864565:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d180 x1877686537434880/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701667 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701651.864582:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acc400. 02000000:00000010:3.0:1790701651.864586:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441c00. 00000100:00000010:3.0:1790701651.864589:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d180. 00000100:00000010:3.0:1790701653.203647:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e48a0. 00000100:00000010:2.0:1790701653.210867:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e43c0. 00000001:02000400:1.0:1790701655.973092:0:132499:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 604: lnet_legacy2yaml: default hop with explicit value ========================================================== 13:07:34 (1790701654) 00000100:00000010:0.0:1790701656.972667:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e0700. 02000000:00000010:0.0:1790701656.972694:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285300. 00000100:00000010:0.0:1790701656.972735:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e1880. 02000000:00000010:0.0:1790701656.972740:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285f00. 00000100:00000010:0.0:1790701656.972747:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e2a00. 02000000:00000010:0.0:1790701656.972751:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285100. 02000000:00000010:0.0:1790701656.972789:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd03d5600. 00000400:00000200:0.0:1790701656.972800:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701656.972805:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f058. 00000400:00000010:0.0:1790701656.972811:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbbb0. 00000100:00000200:0.0:1790701656.972837:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435264, portal 25 00000400:00000010:0.0:1790701656.972843:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbc38. 00000100:00000200:0.0:1790701656.972847:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537435264, offset 0 00000400:00000200:0.0:1790701656.972857:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701656.972866:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701656.972875:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701656.972881:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701656.972886:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701656.972891:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701656.972895:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701656.972898:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [118:118:256:4294967295] 192.168.204.157@tcp LPNI seq info [118:118:8:4294967295] 00000400:00000200:0.0:1790701656.972903:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701656.972915:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701656.972921:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701656.972926:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285500. 00000800:00000200:0.0:1790701656.972935:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701656.972944:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701656.972959:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701656.973043:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:0.0:1790701656.973049:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772cba0. 00000100:00000010:0.0:1790701656.973065:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772cea0. 02000000:00000010:0.0:1790701656.973079:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd03d5200. 00000400:00000200:0.0:1790701656.973083:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701656.973085:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f370. 00000400:00000010:0.0:1790701656.973089:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb660. 00000100:00000200:0.0:1790701656.973092:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435520, portal 10 00000400:00000010:0.0:1790701656.973099:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb908. 00000100:00000200:0.0:1790701656.973101:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537435520, offset 0 00000400:00000200:0.0:1790701656.973106:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701656.973109:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701656.973112:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701656.973115:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701656.973118:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701656.973120:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701656.973122:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701656.973124:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [119:119:255:4294967295] 192.168.204.157@tcp LPNI seq info [119:119:7:4294967295] 00000400:00000200:0.0:1790701656.973128:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701656.973134:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701656.973138:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701656.973141:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285700. 00000800:00000200:0.0:1790701656.973144:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701656.973148:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701656.973150:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701656.973157:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:0.0:1790701656.973175:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc12e6e00. 00000400:00000200:0.0:1790701656.973178:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701656.973180:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f0b0. 00000400:00000010:0.0:1790701656.973183:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbdd0. 00000100:00000200:0.0:1790701656.973185:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435392, portal 10 00000400:00000010:0.0:1790701656.973189:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbd48. 00000100:00000200:0.0:1790701656.973191:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537435392, offset 0 00000400:00000200:0.0:1790701656.973201:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701656.973205:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701656.973208:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701656.973210:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701656.973213:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701656.973215:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701656.973218:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701656.973220:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [120:120:254:4294967295] 192.168.204.157@tcp LPNI seq info [120:120:6:4294967295] 00000400:00000200:0.0:1790701656.973224:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701656.973229:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701656.973234:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701656.973239:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285c00. 00000800:00000200:0.0:1790701656.973243:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701656.973247:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701656.973249:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701656.973266:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701656.975643:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701656.975654:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285500. 00000400:00000200:3.0:1790701656.975665:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701656.975674:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701656.975682:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbc38 00000400:00000010:3.0:1790701656.975684:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbc38. 00000100:00000200:3.0:1790701656.975698:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e0700 x1877686537435264/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701656.975724:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701656.975726:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285700. 00000400:00000200:3.0:1790701656.975728:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701656.975731:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701656.975733:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb908 00000400:00000010:3.0:1790701656.975735:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb908. 00000100:00000200:3.0:1790701656.975740:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e2a00 x1877686537435520/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701656.975805:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701656.975808:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285c00. 00000400:00000200:3.0:1790701656.975811:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701656.975814:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701656.975829:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbd48 00000400:00000010:3.0:1790701656.975840:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbd48. 00000100:00000200:3.0:1790701656.975846:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e1880 x1877686537435392/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701656.978761:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.978811:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701656.978841:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.978847:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.978852:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.978857:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103980 00000400:00000200:3.0:1790701656.978862:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.978868:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x785 [1] + 224 00000400:00000200:3.0:1790701656.978881:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701656.978887:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701656.978902:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e2a00 x1877686537435520/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701656.979014:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.979018:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701656.979021:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.979023:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.979026:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.979029:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103900 00000400:00000200:3.0:1790701656.979032:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.979036:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x78d [1] + 224 00000400:00000200:3.0:1790701656.979045:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701656.979048:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701656.979054:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e1880 x1877686537435392/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701656.980312:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.980320:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701656.980338:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.980342:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.980346:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.980349:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103880 00000400:00000200:3.0:1790701656.980353:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701656.980358:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x77d [1] + 224 00000400:00000200:3.0:1790701656.980369:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701656.980373:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701656.980384:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e0700 x1877686537435264/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701656.982160:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f370. 00000400:00000200:0.0:1790701656.982173:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb660 00000400:00000010:0.0:1790701656.982176:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb660. 00000100:00000200:0.0:1790701656.982191:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e2a00 x1877686537435520/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701656.982203:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e2a00 x1877686537435520/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701656.982243:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd03d5200. 02000000:00000010:0.0:1790701656.982256:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285100. 00000100:00000010:0.0:1790701656.982261:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e2a00. 00000400:00000010:0.0:1790701656.982269:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f0b0. 00000400:00000200:0.0:1790701656.982271:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbdd0 00000400:00000010:0.0:1790701656.982273:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbdd0. 00000100:00000200:0.0:1790701656.982277:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e1880 x1877686537435392/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701656.982294:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e1880 x1877686537435392/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701656.982311:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f058. 00000400:00000200:0.0:1790701656.982314:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbbb0 00000400:00000010:0.0:1790701656.982316:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbbb0. 00000100:00000200:0.0:1790701656.982321:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e0700 x1877686537435264/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701656.982330:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e0700 x1877686537435264/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701672 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701656.982341:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc12e6e00. 02000000:00000010:0.0:1790701656.982347:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285f00. 00000100:00000010:0.0:1790701656.982350:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e1880. 02000000:00000010:0.0:1790701656.982354:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd03d5600. 02000000:00000010:0.0:1790701656.982357:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285300. 00000100:00000010:0.0:1790701656.982360:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e0700. 00000100:00000010:3.0:1790701658.252653:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772cea0. 00000100:00000010:2.0:1790701658.261680:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772cba0. 00000100:00000010:2.0:1790701662.092106:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670000. 02000000:00000010:2.0:1790701662.092130:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4500. 00000100:00000010:2.0:1790701662.092175:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672680. 02000000:00000010:2.0:1790701662.092181:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4700. 00000100:00000010:2.0:1790701662.092189:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672a00. 02000000:00000010:2.0:1790701662.092194:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4100. 02000000:00000010:3.0:1790701662.092319:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7ace800. 00000400:00000200:3.0:1790701662.092330:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701662.092334:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8. 00000400:00000010:3.0:1790701662.092341:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060440. 00000100:00000200:3.0:1790701662.092352:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435648, portal 25 00000400:00000010:3.0:1790701662.092358:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701662.092361:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537435648, offset 0 00000400:00000200:3.0:1790701662.092370:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701662.092379:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.092387:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701662.092400:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701662.092473:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701662.092478:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701662.092481:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701662.092485:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [121:121:256:4294967295] 192.168.204.157@tcp LPNI seq info [121:121:8:4294967295] 00000400:00000200:3.0:1790701662.092490:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.092499:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701662.092506:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701662.092513:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441000. 00000800:00000200:3.0:1790701662.092539:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701662.092548:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701662.092553:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701662.092567:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701662.092573:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4840. 00000100:00000010:3.0:1790701662.092586:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e48a0. 02000000:00000010:3.0:1790701662.092598:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acf600. 00000400:00000200:3.0:1790701662.092602:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701662.092604:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318. 00000400:00000010:3.0:1790701662.092608:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060ee0. 00000100:00000200:3.0:1790701662.092611:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435904, portal 10 00000400:00000010:3.0:1790701662.092617:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701662.092619:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537435904, offset 0 00000400:00000200:3.0:1790701662.092624:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701662.092628:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.092632:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701662.092634:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701662.092637:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701662.092639:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701662.092641:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701662.092644:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [122:122:255:4294967295] 192.168.204.157@tcp LPNI seq info [122:122:7:4294967295] 00000400:00000200:3.0:1790701662.092649:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.092654:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701662.092659:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701662.092663:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441c00. 00000800:00000200:3.0:1790701662.092666:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701662.092670:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701662.092673:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701662.092679:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701662.092700:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acf800. 00000400:00000200:3.0:1790701662.092711:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701662.092713:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b58. 00000400:00000010:3.0:1790701662.092717:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060880. 00000100:00000200:3.0:1790701662.092720:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537435776, portal 10 00000400:00000010:3.0:1790701662.092725:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701662.092728:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537435776, offset 0 00000400:00000200:3.0:1790701662.092732:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701662.092737:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.092740:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701662.092743:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701662.092745:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701662.092748:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701662.092750:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701662.092753:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [123:123:254:4294967295] 192.168.204.157@tcp LPNI seq info [123:123:6:4294967295] 00000400:00000200:3.0:1790701662.092757:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.092763:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701662.092767:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701662.092771:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441500. 00000800:00000200:3.0:1790701662.092775:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701662.092779:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701662.092782:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701662.092788:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701662.092886:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701662.092891:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441000. 00000400:00000200:3.0:1790701662.092897:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701662.092902:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701662.092908:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:3.0:1790701662.092910:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701662.092920:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670000 x1877686537435648/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701662.093276:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701662.093285:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441c00. 00000400:00000200:3.0:1790701662.093288:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701662.093293:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701662.093296:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:3.0:1790701662.093298:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701662.093303:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672a00 x1877686537435904/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701662.093739:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701662.093743:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441500. 00000400:00000200:3.0:1790701662.093747:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701662.093751:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701662.093755:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060aa0 00000400:00000010:3.0:1790701662.093757:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701662.094561:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672680 x1877686537435776/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701662.097471:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097482:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701662.097491:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097496:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097502:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097506:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103a00 00000400:00000200:3.0:1790701662.097511:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097517:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x795 [1] + 224 00000400:00000200:3.0:1790701662.097529:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701662.097534:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701662.097551:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670000 x1877686537435648/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701662.097596:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097600:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701662.097604:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097622:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097626:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097628:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103b00 00000400:00000200:3.0:1790701662.097631:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097636:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x7a5 [1] + 224 00000400:00000200:3.0:1790701662.097642:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701662.097646:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701662.097653:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672a00 x1877686537435904/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701662.097667:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097670:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701662.097674:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097676:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097679:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097683:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103a80 00000400:00000200:3.0:1790701662.097686:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701662.097690:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x7b5 [1] + 224 00000400:00000200:3.0:1790701662.097698:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701662.097702:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701662.097708:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672680 x1877686537435776/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701662.099267:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b58. 00000400:00000200:2.0:1790701662.099281:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060880 00000400:00000010:2.0:1790701662.099284:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060880. 00000100:00000200:2.0:1790701662.099297:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672680 x1877686537435776/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701662.099308:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672680 x1877686537435776/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701662.099346:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8. 00000400:00000200:2.0:1790701662.099349:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060440 00000400:00000010:2.0:1790701662.099351:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060440. 00000100:00000200:2.0:1790701662.099356:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670000 x1877686537435648/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701662.099365:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670000 x1877686537435648/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701662.099378:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318. 00000400:00000200:2.0:1790701662.099407:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060ee0 00000400:00000010:2.0:1790701662.099409:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060ee0. 00000100:00000200:2.0:1790701662.099416:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672a00 x1877686537435904/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701662.099424:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672a00 x1877686537435904/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701678 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701662.099443:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acf800. 02000000:00000010:2.0:1790701662.099450:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4700. 00000100:00000010:2.0:1790701662.099455:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672680. 02000000:00000010:2.0:1790701662.099459:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7ace800. 02000000:00000010:2.0:1790701662.099462:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4500. 00000100:00000010:2.0:1790701662.099464:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670000. 02000000:00000010:2.0:1790701662.099466:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acf600. 02000000:00000010:2.0:1790701662.099468:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4100. 00000100:00000010:2.0:1790701662.099470:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672a00. 00000100:00000010:2.0:1790701663.373679:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4840. 00000100:00000010:3.0:1790701663.374027:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e48a0. 00000001:02000400:1.0:1790701664.618067:0:133079:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 605: lnet_legacy2yaml: quote leading star in IP pattern ========================================================== 13:07:43 (1790701663) 00000100:00000010:0.0:1790701667.225152:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e0700. 02000000:00000010:0.0:1790701667.225177:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285d00. 00000100:00000010:0.0:1790701667.225219:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e1880. 02000000:00000010:0.0:1790701667.225224:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285000. 00000100:00000010:0.0:1790701667.225233:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e2a00. 02000000:00000010:0.0:1790701667.225237:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285800. 02000000:00000010:0.0:1790701667.225279:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccc708200. 00000400:00000200:0.0:1790701667.225293:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701667.225298:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f058. 00000400:00000010:0.0:1790701667.225307:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbbb0. 00000100:00000200:0.0:1790701667.225318:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436032, portal 25 00000400:00000010:0.0:1790701667.225324:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbdd0. 00000100:00000200:0.0:1790701667.225329:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537436032, offset 0 00000400:00000200:0.0:1790701667.225340:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701667.225359:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701667.225368:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701667.225375:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701667.225380:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701667.225384:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701667.225387:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701667.225391:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [124:124:256:4294967295] 192.168.204.157@tcp LPNI seq info [124:124:8:4294967295] 00000400:00000200:0.0:1790701667.225396:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701667.225408:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701667.225416:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701667.225421:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285900. 00000800:00000200:0.0:1790701667.225429:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701667.225439:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701667.225444:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701667.225461:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:0.0:1790701667.225467:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772cd80. 00000100:00000010:0.0:1790701667.225480:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772ca20. 02000000:00000010:0.0:1790701667.225494:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cfe2c8000. 00000400:00000200:0.0:1790701667.225498:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701667.225501:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f0b0. 00000400:00000010:0.0:1790701667.225505:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb660. 00000100:00000200:0.0:1790701667.225509:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436288, portal 10 00000400:00000010:0.0:1790701667.225516:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb6e8. 00000100:00000200:0.0:1790701667.225519:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537436288, offset 0 00000400:00000200:0.0:1790701667.225524:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701667.225527:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701667.225531:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701667.225534:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701667.225536:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701667.225538:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701667.225541:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701667.225550:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [125:125:255:4294967295] 192.168.204.157@tcp LPNI seq info [125:125:7:4294967295] 00000400:00000200:0.0:1790701667.225554:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701667.225561:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701667.225565:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701667.225569:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285b00. 00000800:00000200:0.0:1790701667.225573:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701667.225577:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701667.225580:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701667.225587:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:0.0:1790701667.225611:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e91800. 00000400:00000200:0.0:1790701667.225615:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701667.225618:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f370. 00000400:00000010:0.0:1790701667.225622:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701667.225625:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436160, portal 10 00000400:00000010:0.0:1790701667.225629:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb440. 00000100:00000200:0.0:1790701667.225632:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537436160, offset 0 00000400:00000200:0.0:1790701667.225636:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701667.225641:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701667.225644:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701667.225647:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701667.225649:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701667.225651:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701667.225653:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701667.225656:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [126:126:254:4294967295] 192.168.204.157@tcp LPNI seq info [126:126:6:4294967295] 00000400:00000200:0.0:1790701667.225659:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701667.225665:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701667.225669:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701667.225672:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285c00. 00000800:00000200:0.0:1790701667.225675:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701667.225679:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701667.225687:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701667.225693:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701667.234586:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701667.234600:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285900. 00000400:00000200:2.0:1790701667.234608:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701667.234622:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701667.234633:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbdd0 00000400:00000010:2.0:1790701667.234638:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbdd0. 00000100:00000200:2.0:1790701667.234655:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e0700 x1877686537436032/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701667.234687:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701667.234689:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285b00. 00000400:00000200:2.0:1790701667.234692:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701667.234695:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701667.234697:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb6e8 00000400:00000010:2.0:1790701667.234699:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb6e8. 00000100:00000200:2.0:1790701667.234704:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e2a00 x1877686537436288/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701667.234784:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701667.234787:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285c00. 00000400:00000200:2.0:1790701667.234790:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701667.234795:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701667.234798:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb440 00000400:00000010:2.0:1790701667.234800:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb440. 00000100:00000200:2.0:1790701667.234807:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e1880 x1877686537436160/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701667.239455:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239470:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701667.239480:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239486:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239492:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239497:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103b80 00000400:00000200:3.0:1790701667.239503:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239521:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x7c5 [1] + 224 00000400:00000200:3.0:1790701667.239535:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701667.239540:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701667.239558:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e0700 x1877686537436032/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701667.239588:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239591:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701667.239595:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239598:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239600:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239603:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103c80 00000400:00000200:3.0:1790701667.239607:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239611:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x7cd [1] + 224 00000400:00000200:3.0:1790701667.239617:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701667.239620:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701667.239626:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e2a00 x1877686537436288/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701667.239638:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239642:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701667.239646:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239648:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239650:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.239653:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103c00 00000400:00000010:0.0:1790701667.239809:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f058. 00000400:00000200:0.0:1790701667.239815:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbbb0 00000400:00000010:0.0:1790701667.239831:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbbb0. 00000100:00000200:0.0:1790701667.239838:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e0700 x1877686537436032/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701667.239848:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e0700 x1877686537436032/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701667.239882:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f0b0. 00000400:00000200:0.0:1790701667.239884:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb660 00000400:00000010:0.0:1790701667.239886:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb660. 00000100:00000200:0.0:1790701667.239890:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e2a00 x1877686537436288/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701667.239898:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e2a00 x1877686537436288/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701667.239925:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccc708200. 02000000:00000010:0.0:1790701667.239934:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285d00. 00000100:00000010:0.0:1790701667.239940:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e0700. 02000000:00000010:0.0:1790701667.239944:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cfe2c8000. 02000000:00000010:0.0:1790701667.239947:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285800. 00000100:00000010:0.0:1790701667.239949:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e2a00. 00000400:00000200:3.0:1790701667.240610:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701667.240618:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x7d5 [1] + 224 00000400:00000200:3.0:1790701667.240639:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701667.240643:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701667.240653:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e1880 x1877686537436160/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701667.250731:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f370. 00000400:00000200:0.0:1790701667.250766:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbcc0 00000400:00000010:0.0:1790701667.250769:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701667.250793:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e1880 x1877686537436160/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701667.250810:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e1880 x1877686537436160/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701683 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701667.250883:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e91800. 02000000:00000010:0.0:1790701667.250915:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285000. 00000100:00000010:0.0:1790701667.250921:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e1880. 00000100:00000010:2.0:1790701668.558393:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772cd80. 00000100:00000010:3.0:1790701668.558688:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772ca20. 00000100:00000010:3.0:1790701672.334533:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d180. 02000000:00000010:3.0:1790701672.334572:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441c00. 00000100:00000010:3.0:1790701672.334615:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d500. 02000000:00000010:3.0:1790701672.334619:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441000. 00000100:00000010:3.0:1790701672.334626:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c000. 02000000:00000010:3.0:1790701672.334629:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441800. 02000000:00000010:3.0:1790701672.335474:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea1000. 00000400:00000200:3.0:1790701672.335493:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701672.335499:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223bb0. 00000400:00000010:3.0:1790701672.335508:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060aa0. 00000100:00000200:3.0:1790701672.335536:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436416, portal 25 00000400:00000010:3.0:1790701672.335543:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18. 00000100:00000200:3.0:1790701672.335547:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537436416, offset 0 00000400:00000200:3.0:1790701672.335566:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701672.335575:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701672.335585:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701672.335591:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701672.335596:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701672.335600:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701672.335602:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701672.335606:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [127:127:256:4294967295] 192.168.204.157@tcp LPNI seq info [127:127:8:4294967295] 00000400:00000200:3.0:1790701672.335611:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701672.335623:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701672.335630:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701672.335637:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441900. 00000800:00000200:3.0:1790701672.335645:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701672.335664:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701672.335671:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701672.335710:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701672.335716:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4420. 00000100:00000010:3.0:1790701672.335732:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4cc0. 02000000:00000010:3.0:1790701672.335745:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea1400. 00000400:00000200:3.0:1790701672.335749:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701672.335752:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223c08. 00000400:00000010:3.0:1790701672.335756:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060dd0. 00000100:00000200:3.0:1790701672.335759:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436672, portal 10 00000400:00000010:3.0:1790701672.335765:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060f68. 00000100:00000200:3.0:1790701672.335767:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537436672, offset 0 00000400:00000200:3.0:1790701672.335783:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701672.335787:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701672.335791:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701672.335794:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701672.335796:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701672.335798:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701672.335801:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701672.335803:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [128:128:255:4294967295] 192.168.204.157@tcp LPNI seq info [128:128:7:4294967295] 00000400:00000200:3.0:1790701672.335807:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701672.335813:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701672.335845:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701672.335849:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441400. 00000800:00000200:3.0:1790701672.335854:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701672.335858:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701672.335872:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701672.335879:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701672.335904:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea0c00. 00000400:00000200:3.0:1790701672.335907:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701672.335910:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62230b0. 00000400:00000010:3.0:1790701672.335913:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060e58. 00000100:00000200:3.0:1790701672.335916:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436544, portal 10 00000400:00000010:3.0:1790701672.335929:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060088. 00000100:00000200:3.0:1790701672.335932:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537436544, offset 0 00000400:00000200:3.0:1790701672.335937:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701672.335941:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701672.335945:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701672.335948:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701672.335950:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701672.335953:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701672.335955:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701672.335958:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [129:129:254:4294967295] 192.168.204.157@tcp LPNI seq info [129:129:6:4294967295] 00000400:00000200:3.0:1790701672.335962:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701672.335968:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701672.335972:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701672.335976:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441a00. 00000800:00000200:3.0:1790701672.335980:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701672.335984:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701672.335987:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701672.335992:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701672.338687:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701672.338699:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441900. 00000400:00000200:2.0:1790701672.338709:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701672.338719:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701672.338726:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18 00000400:00000010:2.0:1790701672.338728:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18. 00000100:00000200:2.0:1790701672.338743:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d180 x1877686537436416/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701688 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701672.338776:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701672.338778:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441400. 00000400:00000200:2.0:1790701672.338782:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701672.338786:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701672.338790:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060f68 00000400:00000010:2.0:1790701672.338792:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060f68. 00000100:00000200:2.0:1790701672.338799:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074c000 x1877686537436672/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701672.339236:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701672.339240:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441a00. 00000400:00000200:2.0:1790701672.339244:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701672.339249:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701672.339252:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060088 00000400:00000010:2.0:1790701672.339266:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060088. 00000100:00000200:2.0:1790701672.339273:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074d500 x1877686537436544/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701672.345415:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.345426:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701672.345438:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.345446:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.345453:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.345471:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103d00 00000400:00000200:2.0:1790701672.345477:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.345484:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x7dd [1] + 224 00000400:00000200:2.0:1790701672.345505:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701672.345510:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701672.345528:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d180 x1877686537436416/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701672.345585:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223bb0. 00000400:00000200:2.0:1790701672.345590:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060aa0 00000400:00000010:2.0:1790701672.345593:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060aa0. 00000100:00000200:2.0:1790701672.345599:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d180 x1877686537436416/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701672.345608:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d180 x1877686537436416/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701672.345644:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea1000. 02000000:00000010:2.0:1790701672.345661:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441c00. 00000100:00000010:2.0:1790701672.345667:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d180. 00000400:00000200:2.0:1790701672.354880:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.354891:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701672.354906:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.354911:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.354916:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.354938:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103e00 00000400:00000200:2.0:1790701672.354943:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.354950:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x7ed [1] + 224 00000400:00000200:2.0:1790701672.354961:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701672.354973:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701672.354991:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074c000 x1877686537436672/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701672.355027:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.355038:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701672.355043:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.355046:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.355049:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.355051:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103d80 00000400:00000200:2.0:1790701672.355067:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701672.355077:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x7fd [1] + 224 00000400:00000200:2.0:1790701672.355087:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701672.355091:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701672.355856:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074d500 x1877686537436544/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701672.355905:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62230b0. 00000400:00000200:2.0:1790701672.355909:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060e58 00000400:00000010:2.0:1790701672.355912:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060e58. 00000100:00000200:2.0:1790701672.355918:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074d500 x1877686537436544/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701672.355926:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074d500 x1877686537436544/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701672.355965:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223c08. 00000400:00000200:2.0:1790701672.355974:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060dd0 00000400:00000010:2.0:1790701672.355976:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060dd0. 00000100:00000200:2.0:1790701672.355989:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074c000 x1877686537436672/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701672.355997:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074c000 x1877686537436672/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701688 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701672.356023:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea0c00. 02000000:00000010:2.0:1790701672.356029:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441000. 00000100:00000010:2.0:1790701672.356034:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d500. 02000000:00000010:2.0:1790701672.356045:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea1400. 02000000:00000010:2.0:1790701672.356048:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441800. 00000100:00000010:2.0:1790701672.356050:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c000. 00000100:00000010:3.0:1790701673.612134:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4cc0. 00000100:00000010:2.0:1790701673.615353:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4420. 00000001:02000400:3.0:1790701674.712998:0:133659:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 606: lnet_legacy2yaml: file inputs === 13:07:53 (1790701673) 00000100:00000010:1.0:1790701677.454991:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701677.455020:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5700. 02000000:00000010:1.0:1790701677.455085:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea1400. 00000400:00000200:1.0:1790701677.455099:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701677.455104:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701677.455112:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701677.455122:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436800, portal 25 00000400:00000010:1.0:1790701677.455128:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2880. 00000100:00000200:1.0:1790701677.455132:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537436800, offset 0 00000400:00000200:1.0:1790701677.455144:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701677.455153:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701677.455161:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701677.455166:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701677.455172:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701677.455176:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701677.455178:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701677.455182:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [130:130:256:4294967295] 192.168.204.157@tcp LPNI seq info [130:130:8:4294967295] 00000400:00000200:1.0:1790701677.455187:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701677.455196:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701677.455202:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701677.455207:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5600. 00000800:00000200:1.0:1790701677.455213:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701677.455222:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701677.455226:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701677.455239:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701677.455246:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cdc00. 00000100:00000010:1.0:1790701677.455272:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd060. 00000100:00000010:1.0:1790701677.455316:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701677.455333:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5f00. 02000000:00000010:1.0:1790701677.455356:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea0600. 00000400:00000200:1.0:1790701677.455361:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701677.455364:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701677.455368:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701677.455371:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537436928, portal 10 00000400:00000010:1.0:1790701677.455376:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b22a8. 00000100:00000200:1.0:1790701677.455378:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537436928, offset 0 00000400:00000200:1.0:1790701677.455384:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701677.455388:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701677.455393:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701677.455395:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701677.455398:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701677.455401:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701677.455404:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701677.455406:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [131:131:255:4294967295] 192.168.204.157@tcp LPNI seq info [131:131:7:4294967295] 00000400:00000200:1.0:1790701677.455410:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701677.455415:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701677.455420:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701677.455423:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5200. 00000800:00000200:1.0:1790701677.455427:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701677.455431:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701677.455434:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701677.455440:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701677.455458:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701677.455464:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5000. 02000000:00000010:1.0:1790701677.455488:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccc48ba00. 00000400:00000200:1.0:1790701677.455491:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701677.455493:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701677.455498:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2330. 00000100:00000200:1.0:1790701677.455512:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437056, portal 10 00000400:00000010:1.0:1790701677.455517:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2b28. 00000100:00000200:1.0:1790701677.455520:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537437056, offset 0 00000400:00000200:1.0:1790701677.455525:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701677.455528:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701677.455531:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701677.455534:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701677.455536:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701677.455538:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701677.455539:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701677.455542:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [132:132:254:4294967295] 192.168.204.157@tcp LPNI seq info [132:132:6:4294967295] 00000400:00000200:1.0:1790701677.455545:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701677.455562:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701677.455566:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701677.455569:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5c00. 00000800:00000200:1.0:1790701677.455573:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701677.455576:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701677.455579:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701677.455584:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701677.455967:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701677.455974:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5600. 00000400:00000200:3.0:1790701677.455996:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701677.456004:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701677.456011:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2880 00000400:00000010:3.0:1790701677.456019:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2880. 00000100:00000200:3.0:1790701677.456031:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537436800/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701693 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701677.456063:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701677.456065:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5200. 00000400:00000200:3.0:1790701677.456075:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701677.456086:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701677.456089:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b22a8 00000400:00000010:3.0:1790701677.456092:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b22a8. 00000100:00000200:3.0:1790701677.456097:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537436928/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701677.456372:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701677.456375:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5c00. 00000400:00000200:3.0:1790701677.456380:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701677.456384:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701677.456387:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2b28 00000400:00000010:3.0:1790701677.456389:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2b28. 00000100:00000200:3.0:1790701677.456396:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537437056/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701677.460731:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460744:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701677.460754:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460760:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460766:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460771:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7103e80 00000400:00000200:3.0:1790701677.460777:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460784:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x80d [1] + 224 00000400:00000200:3.0:1790701677.460797:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701677.460802:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701677.460844:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537436800/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701677.460879:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460883:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701677.460887:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460890:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460904:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460923:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103f00 00000400:00000200:3.0:1790701677.460927:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460932:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x815 [1] + 224 00000400:00000200:3.0:1790701677.460940:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701677.460943:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701677.460950:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537436928/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701677.460969:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460973:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701677.460977:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460980:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460984:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.460986:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7103f80 00000400:00000200:3.0:1790701677.461002:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701677.461006:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x81d [1] + 224 00000400:00000200:3.0:1790701677.461016:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701677.461020:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701677.461027:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537437056/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701677.461063:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701677.461070:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701677.461073:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701677.461082:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537436928/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701677.461091:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537436928/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701677.461127:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea0600. 02000000:00000010:1.0:1790701677.461136:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701677.461142:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000010:1.0:1790701677.461171:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701677.461180:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2330 00000400:00000010:1.0:1790701677.461182:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2330. 00000100:00000200:1.0:1790701677.461188:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537437056/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701677.461197:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537437056/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701677.461213:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701677.461215:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b25d8 00000400:00000010:1.0:1790701677.461218:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701677.461221:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537436800/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701677.461229:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537436800/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701693 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701677.461239:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccc48ba00. 02000000:00000010:1.0:1790701677.461243:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5000. 00000100:00000010:1.0:1790701677.461246:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701677.461249:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea1400. 02000000:00000010:1.0:1790701677.461252:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701677.461254:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000100:00000010:3.0:1790701678.731885:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd060. 00000100:00000010:2.0:1790701678.732126:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cdc00. 00000100:00000010:1.0:1790701682.573624:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701682.573650:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701682.573682:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701682.573686:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701682.573692:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701682.573696:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5100. 02000000:00000010:1.0:1790701682.573731:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928c00. 00000400:00000200:1.0:1790701682.573741:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701682.573745:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701682.573751:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701682.573759:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437184, portal 25 00000400:00000010:1.0:1790701682.573771:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2330. 00000100:00000200:1.0:1790701682.573775:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537437184, offset 0 00000400:00000200:1.0:1790701682.573785:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701682.573794:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701682.573802:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701682.573807:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701682.573812:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701682.573816:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701682.573848:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701682.573853:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [133:133:256:4294967295] 192.168.204.157@tcp LPNI seq info [133:133:8:4294967295] 00000400:00000200:1.0:1790701682.573858:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701682.573868:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701682.573875:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701682.573881:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5b00. 00000800:00000200:1.0:1790701682.573888:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701682.573895:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701682.573900:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701682.573923:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701682.573928:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cda80. 00000100:00000010:1.0:1790701682.573938:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd8a0. 02000000:00000010:1.0:1790701682.573949:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929200. 00000400:00000200:1.0:1790701682.573952:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701682.573955:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701682.573958:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701682.573961:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437440, portal 10 00000400:00000010:1.0:1790701682.573967:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b24c8. 00000100:00000200:1.0:1790701682.573969:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537437440, offset 0 00000400:00000200:1.0:1790701682.573973:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701682.573986:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701682.573990:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701682.573993:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701682.574002:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701682.574004:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701682.574006:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701682.574009:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [134:134:255:4294967295] 192.168.204.157@tcp LPNI seq info [134:134:7:4294967295] 00000400:00000200:1.0:1790701682.574014:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701682.574020:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701682.574185:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701682.574189:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5500. 00000800:00000200:1.0:1790701682.574194:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701682.574198:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701682.574201:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701682.574207:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701682.574226:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319ae00. 00000400:00000200:1.0:1790701682.574230:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701682.574232:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701682.574235:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701682.574238:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437312, portal 10 00000400:00000010:1.0:1790701682.574242:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2550. 00000100:00000200:1.0:1790701682.574244:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537437312, offset 0 00000400:00000200:1.0:1790701682.574248:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701682.574252:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701682.574255:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701682.574258:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701682.574260:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701682.574263:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701682.574265:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701682.574268:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [135:135:254:4294967295] 192.168.204.157@tcp LPNI seq info [135:135:6:4294967295] 00000400:00000200:1.0:1790701682.574271:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701682.574277:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701682.574281:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701682.574290:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5c00. 00000800:00000200:1.0:1790701682.574294:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701682.574298:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701682.574301:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701682.574306:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701682.574418:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701682.574423:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5b00. 00000400:00000200:2.0:1790701682.574429:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701682.574434:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701682.574440:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2330 00000400:00000010:2.0:1790701682.574442:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2330. 00000100:00000200:2.0:1790701682.574471:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537437184/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701698 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701682.574495:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701682.574497:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5500. 00000400:00000200:2.0:1790701682.574500:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701682.574503:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701682.574505:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b24c8 00000400:00000010:2.0:1790701682.574507:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b24c8. 00000100:00000200:2.0:1790701682.574511:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537437440/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701682.574572:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701682.574575:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5c00. 00000400:00000200:2.0:1790701682.574578:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701682.574588:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701682.574591:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2550 00000400:00000010:2.0:1790701682.574592:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2550. 00000100:00000200:2.0:1790701682.574597:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537437312/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701682.581230:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581240:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701682.581248:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581253:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581349:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581353:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104100 00000400:00000200:3.0:1790701682.581359:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581365:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x82d [1] + 224 00000400:00000200:3.0:1790701682.581377:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701682.581391:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701682.581409:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537437440/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701682.581435:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581438:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701682.581442:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581445:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581460:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581462:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104080 00000400:00000200:3.0:1790701682.581466:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581469:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x835 [1] + 224 00000400:00000200:3.0:1790701682.581476:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701682.581479:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701682.581484:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537437312/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701682.581498:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581501:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701682.581505:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581508:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581511:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581513:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104000 00000400:00000200:3.0:1790701682.581517:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701682.581531:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x825 [1] + 224 00000400:00000200:3.0:1790701682.581539:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701682.581542:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701682.581548:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537437184/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701682.581637:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701682.581642:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2198 00000400:00000010:1.0:1790701682.581644:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701682.581651:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537437312/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701682.581660:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537437312/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701682.581694:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701682.581697:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b25d8 00000400:00000010:1.0:1790701682.581699:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701682.581703:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537437184/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701682.581711:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537437184/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701682.581723:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701682.581725:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701682.581727:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701682.581730:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537437440/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701682.581738:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537437440/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701698 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701682.581754:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319ae00. 02000000:00000010:1.0:1790701682.581759:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701682.581765:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701682.581777:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928c00. 02000000:00000010:1.0:1790701682.581781:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701682.581784:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701682.581787:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929200. 02000000:00000010:1.0:1790701682.581790:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701682.581792:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000100:00000010:2.0:1790701683.853024:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cda80. 00000100:00000010:3.0:1790701683.855891:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd8a0. 00000001:02000400:0.0:1790701685.248342:0:134241:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 607: lnet_legacy2yaml: options line with combined ip2nets and routes ========================================================== 13:08:03 (1790701683) 00000100:00000010:1.0:1790701687.694546:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701687.694575:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701687.694612:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701687.694617:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701687.694624:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701687.694627:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5100. 02000000:00000010:1.0:1790701687.694668:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056400. 00000400:00000200:1.0:1790701687.694678:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701687.694682:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701687.694691:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701687.694699:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437568, portal 25 00000400:00000010:1.0:1790701687.694704:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701687.694708:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537437568, offset 0 00000400:00000200:1.0:1790701687.694717:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701687.694725:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701687.694733:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701687.694748:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701687.694754:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701687.694758:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701687.694761:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701687.694765:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [136:136:256:4294967295] 192.168.204.157@tcp LPNI seq info [136:136:8:4294967295] 00000400:00000200:1.0:1790701687.694769:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701687.694780:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701687.694787:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701687.694792:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5200. 00000800:00000200:1.0:1790701687.694809:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701687.694845:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701687.694850:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701687.694862:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701687.694868:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd300. 00000100:00000010:1.0:1790701687.694878:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd6c0. 02000000:00000010:1.0:1790701687.694890:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056200. 00000400:00000200:1.0:1790701687.694893:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701687.694895:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701687.694899:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701687.694902:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437824, portal 10 00000400:00000010:1.0:1790701687.694908:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2660. 00000100:00000200:1.0:1790701687.694910:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537437824, offset 0 00000400:00000200:1.0:1790701687.694915:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701687.694919:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701687.694922:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701687.694925:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701687.694928:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701687.694929:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701687.694932:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701687.694934:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [137:137:255:4294967295] 192.168.204.157@tcp LPNI seq info [137:137:7:4294967295] 00000400:00000200:1.0:1790701687.694937:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701687.694943:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701687.694947:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701687.694950:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5600. 00000800:00000200:1.0:1790701687.694953:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701687.694957:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701687.694960:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701687.694965:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701687.694985:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1057800. 00000400:00000200:1.0:1790701687.694993:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701687.694995:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701687.694999:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2908. 00000100:00000200:1.0:1790701687.695001:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437696, portal 10 00000400:00000010:1.0:1790701687.695006:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2c38. 00000100:00000200:1.0:1790701687.695008:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537437696, offset 0 00000400:00000200:1.0:1790701687.695012:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701687.695016:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701687.695018:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701687.695020:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701687.695022:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701687.695024:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701687.695026:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701687.695028:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [138:138:254:4294967295] 192.168.204.157@tcp LPNI seq info [138:138:6:4294967295] 00000400:00000200:1.0:1790701687.695031:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701687.695036:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701687.695040:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701687.695043:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5d00. 00000800:00000200:1.0:1790701687.695046:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701687.695050:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701687.695052:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701687.695056:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701687.695160:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701687.695165:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5200. 00000400:00000200:2.0:1790701687.695173:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701687.695181:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701687.695187:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b25d8 00000400:00000010:2.0:1790701687.695190:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b25d8. 00000100:00000200:2.0:1790701687.695201:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537437568/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701703 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701687.695286:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701687.695308:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5600. 00000400:00000200:3.0:1790701687.695313:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701687.695322:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701687.695325:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2660 00000400:00000010:3.0:1790701687.695327:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2660. 00000100:00000200:3.0:1790701687.695334:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537437824/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701687.695424:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701687.695428:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5d00. 00000400:00000200:3.0:1790701687.695431:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701687.695435:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701687.695438:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2c38 00000400:00000010:3.0:1790701687.695440:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2c38. 00000100:00000200:3.0:1790701687.695445:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537437696/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701687.701957:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.701968:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701687.701976:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.701981:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.701987:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.701991:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104180 00000400:00000200:2.0:1790701687.701996:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.702002:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x83d [1] + 224 00000400:00000200:2.0:1790701687.702015:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701687.702020:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701687.702037:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537437568/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701687.702115:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701687.702120:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701687.702123:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701687.702130:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537437568/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701687.702138:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537437568/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701687.702169:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056400. 02000000:00000010:1.0:1790701687.702189:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701687.702194:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000200:2.0:1790701687.710575:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710586:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701687.710596:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710601:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710620:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710625:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104280 00000400:00000200:2.0:1790701687.710630:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710637:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x845 [1] + 224 00000400:00000200:2.0:1790701687.710650:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701687.710655:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701687.710673:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537437824/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701687.710710:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710714:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701687.710718:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710720:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710723:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710725:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104200 00000400:00000200:2.0:1790701687.710728:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701687.710732:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x84d [1] + 224 00000400:00000200:2.0:1790701687.710751:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701687.710754:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701687.710760:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537437696/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701687.710859:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701687.710865:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2908 00000400:00000010:1.0:1790701687.710868:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2908. 00000100:00000200:1.0:1790701687.710874:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537437696/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701687.710883:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537437696/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701687.710917:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701687.710920:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2198 00000400:00000010:1.0:1790701687.710922:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701687.710926:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537437824/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701687.710932:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537437824/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701703 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701687.710948:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1057800. 02000000:00000010:1.0:1790701687.710954:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701687.710958:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701687.710963:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056200. 02000000:00000010:1.0:1790701687.710965:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701687.710967:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000100:00000010:2.0:1790701688.971918:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd300. 00000100:00000010:3.0:1790701688.973103:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd6c0. 00000100:00000010:1.0:1790701692.813706:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701692.813979:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701692.814013:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701692.814018:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701692.814024:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701692.814027:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5700. 02000000:00000010:1.0:1790701692.814061:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acf800. 00000400:00000200:1.0:1790701692.814071:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701692.814226:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701692.814233:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701692.814241:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537437952, portal 25 00000400:00000010:1.0:1790701692.814247:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2908. 00000100:00000200:1.0:1790701692.814261:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537437952, offset 0 00000400:00000200:1.0:1790701692.814271:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701692.814279:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701692.814287:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701692.814292:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701692.814297:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701692.814301:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701692.814303:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701692.814307:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [139:139:256:4294967295] 192.168.204.157@tcp LPNI seq info [139:139:8:4294967295] 00000400:00000200:1.0:1790701692.814312:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701692.814322:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701692.814329:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701692.814333:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5900. 00000800:00000200:1.0:1790701692.814629:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701692.814638:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701692.814642:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701692.814942:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701692.814950:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd540. 00000100:00000010:1.0:1790701692.814960:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cdc60. 02000000:00000010:1.0:1790701692.814973:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7ace600. 00000400:00000200:1.0:1790701692.814978:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701692.814980:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701692.814984:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701692.814987:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438208, portal 10 00000400:00000010:1.0:1790701692.814995:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2c38. 00000100:00000200:1.0:1790701692.814998:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537438208, offset 0 00000400:00000200:1.0:1790701692.815003:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701692.815012:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701692.815017:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701692.815020:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701692.815023:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701692.815025:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701692.815028:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701692.815031:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [140:140:255:4294967295] 192.168.204.157@tcp LPNI seq info [140:140:7:4294967295] 00000400:00000200:1.0:1790701692.815035:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701692.815042:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701692.815046:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701692.815050:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5800. 00000800:00000200:1.0:1790701692.815054:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701692.815058:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701692.815061:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701692.815067:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701692.816202:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc2a6d400. 00000400:00000200:1.0:1790701692.816213:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701692.816217:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701692.816222:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2660. 00000100:00000200:1.0:1790701692.816227:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438080, portal 10 00000400:00000010:1.0:1790701692.816233:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b25d8. 00000100:00000200:1.0:1790701692.816235:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537438080, offset 0 00000400:00000200:1.0:1790701692.816242:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701692.816248:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701692.816265:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701692.816270:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701692.816273:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701692.816277:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701692.816279:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701692.816282:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [141:141:254:4294967295] 192.168.204.157@tcp LPNI seq info [141:141:6:4294967295] 00000400:00000200:1.0:1790701692.816286:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701692.816302:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701692.816308:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701692.816312:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5400. 00000800:00000200:1.0:1790701692.816318:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701692.816325:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701692.816328:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701692.816335:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701692.820155:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701692.820166:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5900. 00000400:00000200:2.0:1790701692.820174:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701692.820187:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701692.820196:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2908 00000400:00000010:2.0:1790701692.820199:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2908. 00000100:00000200:2.0:1790701692.820214:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537437952/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701692.820238:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701692.820261:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5800. 00000400:00000200:2.0:1790701692.820272:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701692.820277:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701692.820280:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2c38 00000400:00000010:2.0:1790701692.820282:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2c38. 00000100:00000200:2.0:1790701692.820288:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537438208/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701692.822487:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701692.822496:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5400. 00000400:00000200:2.0:1790701692.822504:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701692.822512:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701692.822519:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b25d8 00000400:00000010:2.0:1790701692.822521:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b25d8. 00000100:00000200:2.0:1790701692.822531:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537438080/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701692.832127:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.832137:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701692.832146:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.832151:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.832156:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.832160:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104400 00000400:00000200:2.0:1790701692.832166:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.832172:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x85d [1] + 224 00000400:00000200:2.0:1790701692.832183:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701692.832188:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701692.832205:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537438208/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701692.832244:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.832247:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701692.832264:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701692.832399:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701692.832406:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701692.832408:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701692.832416:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537438208/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701692.832426:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537438208/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701692.832462:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7ace600. 02000000:00000010:1.0:1790701692.832469:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701692.832474:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000200:2.0:1790701692.837232:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837240:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837247:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104380 00000400:00000200:2.0:1790701692.837264:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837274:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x865 [1] + 224 00000400:00000200:2.0:1790701692.837292:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701692.837298:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701692.837314:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537438080/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701692.837347:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837351:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701692.837358:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837361:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837363:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837366:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104300 00000400:00000200:2.0:1790701692.837370:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701692.837373:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x855 [1] + 224 00000400:00000200:2.0:1790701692.837383:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701692.837386:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701692.837392:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537437952/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701692.839539:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701692.839889:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2660 00000400:00000010:1.0:1790701692.839893:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2660. 00000100:00000200:1.0:1790701692.839910:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537438080/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701692.839923:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537438080/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701692.839967:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701692.839970:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2198 00000400:00000010:1.0:1790701692.839973:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701692.839978:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537437952/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701692.840007:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537437952/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701708 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701692.840026:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc2a6d400. 02000000:00000010:1.0:1790701692.840033:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701692.840039:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701692.840043:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acf800. 02000000:00000010:1.0:1790701692.840046:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701692.840048:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000100:00000010:2.0:1790701694.093982:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd540. 00000100:00000010:3.0:1790701694.096062:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cdc60. 00000001:02000400:3.0:1790701695.187049:0:134821:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 608: lnet_legacy2yaml: error on invalid rule (missing IP pattern) ========================================================== 13:08:13 (1790701693) 00000100:00000010:1.0:1790701697.932283:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701697.932305:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701697.932338:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701697.932344:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701697.932353:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701697.932358:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5700. 02000000:00000010:1.0:1790701697.932394:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e91800. 00000400:00000200:1.0:1790701697.932405:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701697.932409:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701697.932416:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701697.932424:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438336, portal 25 00000400:00000010:1.0:1790701697.932429:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2660. 00000100:00000200:1.0:1790701697.932434:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537438336, offset 0 00000400:00000200:1.0:1790701697.932443:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701697.932451:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701697.932459:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701697.932464:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701697.932469:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701697.932474:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701697.932476:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701697.932480:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [142:142:256:4294967295] 192.168.204.157@tcp LPNI seq info [142:142:8:4294967295] 00000400:00000200:1.0:1790701697.932484:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701697.932502:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701697.932509:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701697.932514:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5300. 00000800:00000200:1.0:1790701697.932521:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701697.932529:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701697.932534:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701697.934218:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701697.934228:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd840. 00000100:00000010:1.0:1790701697.934240:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd4e0. 02000000:00000010:1.0:1790701697.934263:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e91400. 00000400:00000200:1.0:1790701697.934268:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701697.934271:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701697.934278:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701697.934283:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438592, portal 10 00000400:00000010:1.0:1790701697.934290:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2550. 00000100:00000200:1.0:1790701697.934293:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537438592, offset 0 00000400:00000200:1.0:1790701697.934302:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701697.934309:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701697.934316:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701697.934321:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701697.934325:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701697.934329:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701697.934333:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701697.934337:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [143:143:255:4294967295] 192.168.204.157@tcp LPNI seq info [143:143:7:4294967295] 00000400:00000200:1.0:1790701697.934342:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701697.934351:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701697.934357:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701697.934362:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5e00. 00000800:00000200:1.0:1790701697.934369:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701697.934376:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701697.934389:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701697.934397:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701697.934421:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e90000. 00000400:00000200:1.0:1790701697.934425:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701697.934428:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701697.934432:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b24c8. 00000100:00000200:1.0:1790701697.934435:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438464, portal 10 00000400:00000010:1.0:1790701697.934439:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2330. 00000100:00000200:1.0:1790701697.934441:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537438464, offset 0 00000400:00000200:1.0:1790701697.934446:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701697.934451:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701697.934454:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701697.934457:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701697.934459:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701697.934462:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701697.934464:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701697.934466:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [144:144:254:4294967295] 192.168.204.157@tcp LPNI seq info [144:144:6:4294967295] 00000400:00000200:1.0:1790701697.934470:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701697.934477:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701697.934481:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701697.934485:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5a00. 00000800:00000200:1.0:1790701697.934488:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701697.934492:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701697.934495:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701697.934500:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701697.939941:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701697.939963:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5300. 00000400:00000200:3.0:1790701697.939970:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701697.939980:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701697.939987:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2660 00000400:00000010:3.0:1790701697.939990:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2660. 00000100:00000200:3.0:1790701697.940004:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537438336/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701697.940029:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701697.940032:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5e00. 00000400:00000200:3.0:1790701697.940035:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701697.940039:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701697.940041:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2550 00000400:00000010:3.0:1790701697.940051:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2550. 00000100:00000200:3.0:1790701697.940057:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537438592/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701697.940122:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701697.940125:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5a00. 00000400:00000200:3.0:1790701697.940128:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701697.940132:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701697.940134:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2330 00000400:00000010:3.0:1790701697.940136:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2330. 00000100:00000200:3.0:1790701697.940141:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537438464/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701697.957348:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957360:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701697.957369:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957374:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957380:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957384:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104480 00000400:00000200:2.0:1790701697.957389:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957395:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x86d [1] + 224 00000400:00000200:2.0:1790701697.957416:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701697.957422:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701697.957438:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537438336/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701697.957469:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957472:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701697.957476:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957478:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957481:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957484:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104580 00000400:00000200:2.0:1790701697.957487:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957491:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x875 [1] + 224 00000400:00000200:2.0:1790701697.957497:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701697.957501:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701697.957507:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537438592/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701697.957522:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957526:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701697.957530:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957533:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.957536:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701697.957916:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:1.0:1790701697.957922:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2198 00000400:00000010:1.0:1790701697.957924:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701697.957932:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537438336/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701697.957942:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537438336/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701697.957978:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:1.0:1790701697.957981:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:1.0:1790701697.957982:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701697.957987:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537438592/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701697.957996:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537438592/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701697.958016:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e91800. 02000000:00000010:1.0:1790701697.958022:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5f00. 00000100:00000010:1.0:1790701697.958028:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701697.958033:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e91400. 02000000:00000010:1.0:1790701697.958035:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5700. 00000100:00000010:1.0:1790701697.958038:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000200:2.0:1790701697.958039:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104500 00000400:00000200:2.0:1790701697.958045:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701697.958051:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x87d [1] + 224 00000400:00000200:2.0:1790701697.958066:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701697.958070:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701697.958078:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537438464/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701697.958225:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:1.0:1790701697.958229:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b24c8 00000400:00000010:1.0:1790701697.958231:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b24c8. 00000100:00000200:1.0:1790701697.958237:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537438464/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701697.958246:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537438464/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701713 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701697.958262:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e90000. 02000000:00000010:1.0:1790701697.958268:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701697.958271:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000100:00000010:2.0:1790701699.213959:0:120878:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd840. 00000100:00000010:3.0:1790701699.214217:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd4e0. 00000001:02000400:0.0:1790701703.030362:0:135401:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 609: lnet_legacy2yaml: error on no input ========================================================== 13:08:21 (1790701701) 00000100:00000010:0.0:1790701703.052267:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e1880. 02000000:00000010:0.0:1790701703.052291:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285700. 00000100:00000010:0.0:1790701703.052506:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e2a00. 02000000:00000010:0.0:1790701703.052514:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285200. 00000100:00000010:0.0:1790701703.052522:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e0700. 02000000:00000010:0.0:1790701703.052526:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe285c00. 02000000:00000010:1.0:1790701703.052608:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc27ece00. 00000400:00000200:1.0:1790701703.052619:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701703.052624:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185528. 00000400:00000010:1.0:1790701703.052631:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b24c8. 00000100:00000200:1.0:1790701703.052638:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438720, portal 25 00000400:00000010:1.0:1790701703.052644:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2aa0. 00000100:00000200:1.0:1790701703.052655:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537438720, offset 0 00000400:00000200:1.0:1790701703.052665:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701703.052672:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701703.052679:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701703.052685:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701703.052689:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701703.052693:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701703.052696:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701703.052699:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [145:145:256:4294967295] 192.168.204.157@tcp LPNI seq info [145:145:8:4294967295] 00000400:00000200:1.0:1790701703.052703:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701703.052714:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701703.052719:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701703.052725:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5a00. 00000800:00000200:1.0:1790701703.052731:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701703.052739:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701703.052743:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701703.052756:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701703.052762:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd660. 00000100:00000010:1.0:1790701703.052771:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cde40. 02000000:00000010:1.0:1790701703.052784:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccc7cf600. 00000400:00000200:1.0:1790701703.052787:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701703.052790:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185b00. 00000400:00000010:1.0:1790701703.052794:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2198. 00000100:00000200:1.0:1790701703.052796:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438976, portal 10 00000400:00000010:1.0:1790701703.052802:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2b28. 00000100:00000200:1.0:1790701703.052804:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537438976, offset 0 00000400:00000200:1.0:1790701703.052808:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701703.052811:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701703.052814:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701703.052914:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701703.052917:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701703.052926:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701703.052928:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701703.052931:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [146:146:255:4294967295] 192.168.204.157@tcp LPNI seq info [146:146:7:4294967295] 00000400:00000200:1.0:1790701703.052935:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701703.052942:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701703.052946:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701703.052949:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5e00. 00000800:00000200:1.0:1790701703.052953:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701703.052958:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701703.052961:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701703.052968:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701703.053009:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea0c00. 00000400:00000200:1.0:1790701703.053014:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701703.053017:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185d68. 00000400:00000010:1.0:1790701703.053020:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b22a8. 00000100:00000200:1.0:1790701703.053023:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537438848, portal 10 00000400:00000010:1.0:1790701703.053027:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2880. 00000100:00000200:1.0:1790701703.053030:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537438848, offset 0 00000400:00000200:1.0:1790701703.053035:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701703.053038:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701703.053043:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701703.053046:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701703.053062:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701703.053065:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701703.053068:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701703.053071:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [147:147:254:4294967295] 192.168.204.157@tcp LPNI seq info [147:147:6:4294967295] 00000400:00000200:1.0:1790701703.053076:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701703.053085:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701703.053090:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701703.053094:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5300. 00000800:00000200:1.0:1790701703.053104:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701703.053110:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701703.053112:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701703.053119:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701703.054094:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701703.054101:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5a00. 00000400:00000200:3.0:1790701703.054108:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701703.054115:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701703.054121:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2aa0 00000400:00000010:3.0:1790701703.054123:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2aa0. 00000100:00000200:3.0:1790701703.054134:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e1880 x1877686537438720/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701703.054161:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701703.054163:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5e00. 00000400:00000200:3.0:1790701703.054166:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701703.054170:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701703.054172:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2b28 00000400:00000010:3.0:1790701703.054173:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2b28. 00000100:00000200:3.0:1790701703.054179:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e0700 x1877686537438976/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701703.054248:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701703.054251:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5300. 00000400:00000200:3.0:1790701703.054255:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701703.054265:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701703.054269:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2880 00000400:00000010:3.0:1790701703.054270:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2880. 00000100:00000200:3.0:1790701703.054277:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e2a00 x1877686537438848/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701703.059527:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701703.059544:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701703.059558:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701703.059564:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701703.059572:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701703.059578:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104600 00000400:00000200:2.0:1790701703.059586:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701703.059593:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x885 [1] + 224 00000400:00000200:2.0:1790701703.059628:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701703.059633:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701703.059652:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e1880 x1877686537438720/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701703.059805:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185528. 00000400:00000200:0.0:1790701703.059812:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b24c8 00000400:00000010:0.0:1790701703.059815:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b24c8. 00000100:00000200:0.0:1790701703.059880:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e1880 x1877686537438720/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701703.059892:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e1880 x1877686537438720/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701703.059928:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc27ece00. 02000000:00000010:0.0:1790701703.059937:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285700. 00000100:00000010:0.0:1790701703.059943:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e1880. 00000400:00000200:3.0:1790701703.069491:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069506:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701703.069517:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069525:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069533:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069539:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104700 00000400:00000200:3.0:1790701703.069545:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069555:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x88d [1] + 224 00000400:00000200:3.0:1790701703.069574:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701703.069579:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701703.069597:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e0700 x1877686537438976/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701703.069932:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069938:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701703.069943:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069946:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069950:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069953:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104680 00000400:00000200:3.0:1790701703.069957:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701703.069962:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x895 [1] + 224 00000400:00000200:3.0:1790701703.069977:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701703.069981:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701703.069989:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e2a00 x1877686537438848/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701703.074678:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185b00. 00000400:00000200:0.0:1790701703.074695:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2198 00000400:00000010:0.0:1790701703.074699:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2198. 00000100:00000200:0.0:1790701703.074718:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e0700 x1877686537438976/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701703.074732:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e0700 x1877686537438976/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701703.074775:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccc7cf600. 02000000:00000010:0.0:1790701703.074800:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285c00. 00000100:00000010:0.0:1790701703.074805:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e0700. 00000400:00000010:0.0:1790701703.074840:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185d68. 00000400:00000200:0.0:1790701703.074844:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b22a8 00000400:00000010:0.0:1790701703.074845:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b22a8. 00000100:00000200:0.0:1790701703.074852:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e2a00 x1877686537438848/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701703.074864:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e2a00 x1877686537438848/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701719 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701703.074889:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea0c00. 02000000:00000010:0.0:1790701703.074893:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe285200. 00000100:00000010:0.0:1790701703.074896:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e2a00. 00000100:00000010:2.0:1790701704.336621:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd660. 00000100:00000010:3.0:1790701704.336868:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cde40. 00000100:00000010:2.0:1790701708.174795:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672a00. 02000000:00000010:2.0:1790701708.174846:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4100. 00000100:00000010:2.0:1790701708.174879:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670000. 02000000:00000010:2.0:1790701708.174884:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4800. 00000100:00000010:2.0:1790701708.174892:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672680. 02000000:00000010:2.0:1790701708.174895:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4300. 02000000:00000010:2.0:1790701708.174928:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1057800. 00000400:00000200:2.0:1790701708.174938:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701708.174943:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2210. 00000400:00000010:2.0:1790701708.174961:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a5d8. 00000100:00000200:2.0:1790701708.174971:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439104, portal 25 00000400:00000010:2.0:1790701708.174977:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a2a8. 00000100:00000200:2.0:1790701708.174980:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537439104, offset 0 00000400:00000200:2.0:1790701708.174991:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701708.174999:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701708.175007:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701708.175013:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701708.175019:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701708.175031:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701708.175034:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701708.175040:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [148:148:256:4294967295] 192.168.204.157@tcp LPNI seq info [148:148:8:4294967295] 00000400:00000200:2.0:1790701708.175045:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701708.175057:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701708.175064:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701708.175070:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4200. 00000800:00000200:2.0:1790701708.175077:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701708.175084:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701708.175089:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701708.175103:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:2.0:1790701708.175109:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc9edb900. 00000100:00000010:2.0:1790701708.175120:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc9edb060. 02000000:00000010:2.0:1790701708.175131:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056000. 00000400:00000200:2.0:1790701708.175135:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701708.175137:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b24d0. 00000400:00000010:2.0:1790701708.175141:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a198. 00000100:00000200:2.0:1790701708.175144:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439360, portal 10 00000400:00000010:2.0:1790701708.175150:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701708.175152:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537439360, offset 0 00000400:00000200:2.0:1790701708.175158:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701708.175161:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701708.175176:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701708.175179:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701708.175181:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701708.175183:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701708.175186:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701708.175188:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [149:149:255:4294967295] 192.168.204.157@tcp LPNI seq info [149:149:7:4294967295] 00000400:00000200:2.0:1790701708.175192:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701708.175199:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701708.175212:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701708.175216:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4b00. 00000800:00000200:2.0:1790701708.175220:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701708.175225:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701708.175228:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701708.175234:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:2.0:1790701708.175253:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056200. 00000400:00000200:2.0:1790701708.175257:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701708.175259:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2058. 00000400:00000010:2.0:1790701708.175263:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a990. 00000100:00000200:2.0:1790701708.175266:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439232, portal 10 00000400:00000010:2.0:1790701708.175270:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701708.175272:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537439232, offset 0 00000400:00000200:2.0:1790701708.175276:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701708.175280:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701708.175283:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701708.175285:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701708.175287:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701708.175289:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701708.175291:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701708.175293:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [150:150:254:4294967295] 192.168.204.157@tcp LPNI seq info [150:150:6:4294967295] 00000400:00000200:2.0:1790701708.175296:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701708.175301:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701708.175304:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701708.175308:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4500. 00000800:00000200:2.0:1790701708.175310:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701708.175314:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701708.175316:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701708.175321:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701708.175367:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701708.175376:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4200. 00000400:00000200:2.0:1790701708.175381:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701708.175386:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701708.175391:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a2a8 00000400:00000010:2.0:1790701708.175394:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a2a8. 00000100:00000200:2.0:1790701708.175405:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672a00 x1877686537439104/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701708.175427:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701708.175429:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4b00. 00000400:00000200:2.0:1790701708.175438:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701708.175441:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701708.175444:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a220 00000400:00000010:2.0:1790701708.175446:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701708.175449:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672680 x1877686537439360/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701708.175510:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701708.175513:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4500. 00000400:00000200:2.0:1790701708.175516:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701708.175519:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701708.175522:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a330 00000400:00000010:2.0:1790701708.175523:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701708.175528:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670000 x1877686537439232/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701708.180637:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180649:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701708.180658:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180664:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180670:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180675:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104780 00000400:00000200:3.0:1790701708.180680:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180688:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x89d [1] + 224 00000400:00000200:3.0:1790701708.180701:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701708.180707:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701708.180725:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672a00 x1877686537439104/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701708.180757:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180761:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701708.180766:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180770:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180773:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180777:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104880 00000400:00000200:3.0:1790701708.180780:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180785:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x8ad [1] + 224 00000400:00000200:3.0:1790701708.180793:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701708.180797:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701708.180803:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672680 x1877686537439360/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701708.180839:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180843:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701708.180848:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180859:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180863:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180867:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104800 00000400:00000200:3.0:1790701708.180871:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701708.180874:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x8bd [1] + 224 00000400:00000200:3.0:1790701708.180884:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701708.180887:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701708.180893:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670000 x1877686537439232/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701708.180925:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2058. 00000400:00000200:3.0:1790701708.180930:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a990 00000400:00000010:3.0:1790701708.180932:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a990. 00000100:00000200:3.0:1790701708.180938:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670000 x1877686537439232/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701708.180947:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670000 x1877686537439232/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701708.180993:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2210. 00000400:00000200:3.0:1790701708.180996:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a5d8 00000400:00000010:3.0:1790701708.180998:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a5d8. 00000100:00000200:3.0:1790701708.181003:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672a00 x1877686537439104/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701708.181011:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672a00 x1877686537439104/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701708.181027:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b24d0. 00000400:00000200:3.0:1790701708.181030:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a198 00000400:00000010:3.0:1790701708.181032:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a198. 00000100:00000200:3.0:1790701708.181037:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672680 x1877686537439360/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701708.181352:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672680 x1877686537439360/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701724 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701708.181380:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056200. 02000000:00000010:3.0:1790701708.181386:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4800. 00000100:00000010:3.0:1790701708.181391:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670000. 02000000:00000010:3.0:1790701708.181395:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1057800. 02000000:00000010:3.0:1790701708.181399:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4100. 00000100:00000010:3.0:1790701708.181400:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672a00. 02000000:00000010:3.0:1790701708.181403:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056000. 02000000:00000010:3.0:1790701708.181406:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4300. 00000100:00000010:3.0:1790701708.181408:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672680. 00000100:00000010:2.0:1790701709.452049:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc9edb900. 00000100:00000010:3.0:1790701709.452912:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc9edb060. 00000001:02000400:3.0:1790701711.066907:0:135982:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 610: lnet_legacy2yaml: multiple files and inline, check order ========================================================== 13:08:29 (1790701709) 00000100:00000010:1.0:1790701713.292137:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701713.292162:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5600. 00000100:00000010:1.0:1790701713.292199:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701713.292204:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5200. 00000100:00000010:1.0:1790701713.292210:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701713.292214:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5400. 02000000:00000010:1.0:1790701713.292247:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc37f1600. 00000400:00000200:1.0:1790701713.292257:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701713.292262:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1855d8. 00000400:00000010:1.0:1790701713.292269:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701713.292278:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439488, portal 25 00000400:00000010:1.0:1790701713.292284:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2220. 00000100:00000200:1.0:1790701713.292288:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537439488, offset 0 00000400:00000200:1.0:1790701713.292306:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701713.292316:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701713.292325:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701713.292330:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701713.292335:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701713.292341:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701713.292343:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701713.292347:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [151:151:256:4294967295] 192.168.204.157@tcp LPNI seq info [151:151:8:4294967295] 00000400:00000200:1.0:1790701713.292353:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701713.292363:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701713.292370:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701713.292375:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5d00. 00000800:00000200:1.0:1790701713.292382:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701713.292390:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701713.292405:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701713.292426:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701713.292431:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd360. 00000100:00000010:1.0:1790701713.292442:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd240. 02000000:00000010:1.0:1790701713.292453:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc37f1800. 00000400:00000200:1.0:1790701713.292457:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701713.292460:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1857e8. 00000400:00000010:1.0:1790701713.292464:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2ee0. 00000100:00000200:1.0:1790701713.292467:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439744, portal 10 00000400:00000010:1.0:1790701713.292472:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2000. 00000100:00000200:1.0:1790701713.292475:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537439744, offset 0 00000400:00000200:1.0:1790701713.292479:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701713.292483:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701713.292486:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701713.292489:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701713.292492:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701713.292495:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701713.292498:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701713.292501:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [152:152:255:4294967295] 192.168.204.157@tcp LPNI seq info [152:152:7:4294967295] 00000400:00000200:1.0:1790701713.292505:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701713.292511:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701713.292516:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701713.292520:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5c00. 00000800:00000200:1.0:1790701713.292525:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701713.292529:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701713.292532:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701713.292539:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701713.292558:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc37f0400. 00000400:00000200:1.0:1790701713.292561:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701713.292564:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185ec8. 00000400:00000010:1.0:1790701713.292567:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b23b8. 00000100:00000200:1.0:1790701713.292575:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439616, portal 10 00000400:00000010:1.0:1790701713.292582:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2dd0. 00000100:00000200:1.0:1790701713.292584:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537439616, offset 0 00000400:00000200:1.0:1790701713.292589:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701713.292592:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701713.292596:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701713.292598:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701713.292601:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701713.292603:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701713.292605:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701713.292608:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [153:153:254:4294967295] 192.168.204.157@tcp LPNI seq info [153:153:6:4294967295] 00000400:00000200:1.0:1790701713.292612:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701713.292617:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701713.292621:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701713.292625:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5500. 00000800:00000200:1.0:1790701713.292628:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701713.292631:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701713.292634:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701713.292639:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701713.293404:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701713.293420:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5d00. 00000400:00000200:2.0:1790701713.293428:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701713.293435:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701713.293442:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2220 00000400:00000010:2.0:1790701713.293444:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2220. 00000100:00000200:2.0:1790701713.293455:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537439488/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701713.293497:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701713.293499:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5c00. 00000400:00000200:2.0:1790701713.293502:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701713.293517:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701713.293520:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2000 00000400:00000010:2.0:1790701713.293522:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2000. 00000100:00000200:2.0:1790701713.293527:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537439744/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701713.293608:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701713.293611:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5500. 00000400:00000200:2.0:1790701713.293614:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701713.293618:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701713.293621:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2dd0 00000400:00000010:2.0:1790701713.293623:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2dd0. 00000100:00000200:2.0:1790701713.293629:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537439616/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701713.296738:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296748:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701713.296756:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296761:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296767:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296771:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104a00 00000400:00000200:3.0:1790701713.296776:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296782:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x8d5 [1] + 224 00000400:00000200:3.0:1790701713.296794:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701713.296799:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701713.296815:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537439744/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701713.296932:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296941:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701713.296947:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296949:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296952:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296967:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104980 00000400:00000200:3.0:1790701713.296971:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.296975:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x8dd [1] + 224 00000400:00000200:3.0:1790701713.296985:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701713.296989:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701713.296995:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537439616/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701713.297068:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185ec8. 00000400:00000200:1.0:1790701713.297074:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b23b8 00000400:00000010:1.0:1790701713.297077:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b23b8. 00000100:00000200:1.0:1790701713.297084:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537439616/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701713.297094:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537439616/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701713.297257:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1857e8. 00000400:00000200:1.0:1790701713.297260:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2ee0 00000400:00000010:1.0:1790701713.297261:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2ee0. 00000100:00000200:1.0:1790701713.297266:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537439744/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701713.297280:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537439744/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701713.297297:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc37f0400. 02000000:00000010:1.0:1790701713.297303:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5200. 00000100:00000010:1.0:1790701713.297308:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701713.297313:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc37f1800. 02000000:00000010:1.0:1790701713.297315:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5400. 00000100:00000010:1.0:1790701713.297317:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000400:00000200:3.0:1790701713.300365:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.300377:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701713.300386:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.300393:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.300398:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.300403:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104900 00000400:00000200:3.0:1790701713.300409:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701713.300414:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x8cd [1] + 224 00000400:00000200:3.0:1790701713.300431:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701713.300436:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701713.300452:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537439488/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701713.300519:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1855d8. 00000400:00000200:1.0:1790701713.300524:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2990 00000400:00000010:1.0:1790701713.300527:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701713.300534:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537439488/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701713.300543:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537439488/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701729 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701713.300573:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc37f1600. 02000000:00000010:1.0:1790701713.300578:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5600. 00000100:00000010:1.0:1790701713.300583:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000100:00000010:2.0:1790701714.572539:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd360. 00000100:00000010:3.0:1790701714.572952:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd240. 00000100:00000010:2.0:1790701718.412026:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672680. 02000000:00000010:2.0:1790701718.412051:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4b00. 00000100:00000010:2.0:1790701718.412096:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672a00. 02000000:00000010:2.0:1790701718.412102:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4200. 00000100:00000010:2.0:1790701718.412110:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca670000. 02000000:00000010:2.0:1790701718.412114:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56f4c00. 02000000:00000010:3.0:1790701718.412212:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929600. 00000400:00000200:3.0:1790701718.412224:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701718.412229:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223b00. 00000400:00000010:3.0:1790701718.412236:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060220. 00000100:00000200:3.0:1790701718.412245:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537439872, portal 25 00000400:00000010:3.0:1790701718.412250:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0603b8. 00000100:00000200:3.0:1790701718.412253:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537439872, offset 0 00000400:00000200:3.0:1790701718.412262:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701718.412270:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.412279:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701718.412299:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701718.412303:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701718.412377:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701718.412381:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701718.412385:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [154:154:256:4294967295] 192.168.204.157@tcp LPNI seq info [154:154:8:4294967295] 00000400:00000200:3.0:1790701718.412390:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.412400:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701718.412407:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701718.412413:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441000. 00000800:00000200:3.0:1790701718.412419:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701718.412428:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701718.412432:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701718.412454:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701718.412460:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e46c0. 00000100:00000010:3.0:1790701718.412470:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4ea0. 02000000:00000010:3.0:1790701718.412483:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929000. 00000400:00000200:3.0:1790701718.412486:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701718.412489:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223630. 00000400:00000010:3.0:1790701718.412494:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0605d8. 00000100:00000200:3.0:1790701718.412496:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440128, portal 10 00000400:00000010:3.0:1790701718.412502:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0602a8. 00000100:00000200:3.0:1790701718.412504:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537440128, offset 0 00000400:00000200:3.0:1790701718.412508:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701718.412512:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.412516:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701718.412518:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701718.412521:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701718.412523:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701718.412525:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701718.412528:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [155:155:255:4294967295] 192.168.204.157@tcp LPNI seq info [155:155:7:4294967295] 00000400:00000200:3.0:1790701718.412532:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.412545:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701718.412550:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701718.412553:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441e00. 00000800:00000200:3.0:1790701718.412557:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701718.412561:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701718.412564:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701718.412570:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701718.412591:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928200. 00000400:00000200:3.0:1790701718.412595:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701718.412597:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223840. 00000400:00000010:3.0:1790701718.412600:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060990. 00000100:00000200:3.0:1790701718.412603:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440000, portal 10 00000400:00000010:3.0:1790701718.412608:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0604c8. 00000100:00000200:3.0:1790701718.412610:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537440000, offset 0 00000400:00000200:3.0:1790701718.412614:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701718.412618:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.412622:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701718.412625:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701718.412628:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701718.412630:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701718.412633:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701718.412635:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [156:156:254:4294967295] 192.168.204.157@tcp LPNI seq info [156:156:6:4294967295] 00000400:00000200:3.0:1790701718.412639:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.412645:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701718.412650:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701718.412653:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441200. 00000800:00000200:3.0:1790701718.412657:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701718.412661:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701718.412664:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701718.412669:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701718.412787:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701718.412792:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441000. 00000400:00000200:2.0:1790701718.412798:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701718.412803:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701718.412809:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0603b8 00000400:00000010:2.0:1790701718.412811:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0603b8. 00000100:00000200:2.0:1790701718.412854:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672680 x1877686537439872/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701718.412884:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701718.412887:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441e00. 00000400:00000200:2.0:1790701718.412889:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701718.412893:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701718.412895:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0602a8 00000400:00000010:2.0:1790701718.412897:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0602a8. 00000100:00000200:2.0:1790701718.412901:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca670000 x1877686537440128/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701718.412965:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701718.412968:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441200. 00000400:00000200:2.0:1790701718.412971:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701718.412974:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701718.412977:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0604c8 00000400:00000010:2.0:1790701718.412978:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0604c8. 00000100:00000200:2.0:1790701718.412982:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca672a00 x1877686537440000/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701718.415098:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.415110:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701718.415118:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.415124:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.415129:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.415134:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104b80 00000400:00000200:3.0:1790701718.415139:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.415145:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x8f5 [1] + 224 00000400:00000200:3.0:1790701718.415158:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701718.415162:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701718.415177:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca670000 x1877686537440128/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701718.415544:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223630. 00000400:00000200:2.0:1790701718.415552:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0605d8 00000400:00000010:2.0:1790701718.415554:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0605d8. 00000100:00000200:2.0:1790701718.415562:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca670000 x1877686537440128/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701718.415572:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca670000 x1877686537440128/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701718.415611:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929000. 02000000:00000010:2.0:1790701718.415618:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4c00. 00000100:00000010:2.0:1790701718.415622:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca670000. 00000400:00000200:2.0:1790701718.418705:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701718.418718:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701718.418728:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701718.418764:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701718.418771:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701718.418775:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104b00 00000400:00000200:2.0:1790701718.418780:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701718.418787:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x905 [1] + 224 00000400:00000200:2.0:1790701718.418805:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701718.418810:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701718.418869:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672a00 x1877686537440000/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701718.419178:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223840. 00000400:00000200:3.0:1790701718.419184:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060990 00000400:00000010:3.0:1790701718.419187:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060990. 00000100:00000200:3.0:1790701718.419194:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672a00 x1877686537440000/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701718.419204:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672a00 x1877686537440000/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701718.419243:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928200. 02000000:00000010:3.0:1790701718.419249:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4200. 00000100:00000010:3.0:1790701718.419259:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672a00. 00000400:00000200:3.0:1790701718.422930:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.422941:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701718.422952:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.422958:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.422964:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.422968:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104a80 00000400:00000200:3.0:1790701718.422973:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701718.422979:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x8e5 [1] + 224 00000400:00000200:3.0:1790701718.423092:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701718.423221:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701718.423240:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca672680 x1877686537439872/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701718.423404:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223b00. 00000400:00000200:2.0:1790701718.423412:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060220 00000400:00000010:2.0:1790701718.423415:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060220. 00000100:00000200:2.0:1790701718.423423:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca672680 x1877686537439872/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701718.423433:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca672680 x1877686537439872/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701734 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701718.423465:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929600. 02000000:00000010:2.0:1790701718.423472:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56f4b00. 00000100:00000010:2.0:1790701718.423477:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672680. 00000001:02000400:2.0:1790701718.944085:0:136564:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 611: lnet_legacy2yaml: invalid hopcount ========================================================== 13:08:37 (1790701717) 00000100:00000010:3.0:1790701719.691932:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4ea0. 00000100:00000010:2.0:1790701719.692498:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e46c0. 00000100:00000010:3.0:1790701723.535147:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074fb80. 02000000:00000010:3.0:1790701723.535170:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441200. 02000000:00000010:2.0:1790701723.535299:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928000. 00000400:00000200:2.0:1790701723.535310:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701723.535315:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2dc0. 00000400:00000010:2.0:1790701723.535323:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a330. 00000100:00000200:2.0:1790701723.535330:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440256, portal 25 00000400:00000010:2.0:1790701723.535337:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701723.535340:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537440256, offset 0 00000400:00000200:2.0:1790701723.535349:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701723.535358:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701723.535374:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701723.535379:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701723.535384:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701723.535389:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701723.535391:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701723.535394:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [157:157:256:4294967295] 192.168.204.157@tcp LPNI seq info [157:157:8:4294967295] 00000400:00000200:2.0:1790701723.535398:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701723.535407:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701723.535414:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701723.535418:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4b00. 00000800:00000200:2.0:1790701723.535426:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701723.535432:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701723.535437:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701723.535454:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:2.0:1790701723.535459:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc9edb420. 00000100:00000010:2.0:1790701723.535471:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc9edb240. 00000100:00000010:3.0:1790701723.535507:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ea00. 02000000:00000010:3.0:1790701723.535514:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441800. 02000000:00000010:2.0:1790701723.535551:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928600. 00000400:00000200:2.0:1790701723.535557:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701723.535559:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2580. 00000400:00000010:2.0:1790701723.535562:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a2a8. 00000100:00000200:2.0:1790701723.535565:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440384, portal 10 00000400:00000010:2.0:1790701723.535571:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a088. 00000100:00000200:2.0:1790701723.535572:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537440384, offset 0 00000400:00000200:2.0:1790701723.535577:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701723.535580:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701723.535583:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701723.535586:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701723.535589:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701723.535604:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701723.535607:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701723.535610:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [158:158:255:4294967295] 192.168.204.157@tcp LPNI seq info [158:158:7:4294967295] 00000400:00000200:2.0:1790701723.535620:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701723.535626:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701723.535630:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701723.535633:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4400. 00000800:00000200:2.0:1790701723.535637:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701723.535641:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701723.535644:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701723.535663:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701723.535715:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701723.535719:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4b00. 00000400:00000200:2.0:1790701723.535725:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701723.535730:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701723.535736:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a220 00000400:00000010:2.0:1790701723.535738:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701723.535750:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074fb80 x1877686537440256/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701739 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701723.535855:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701723.535859:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4400. 00000400:00000200:2.0:1790701723.535862:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701723.535865:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701723.535868:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a088 00000400:00000010:2.0:1790701723.535870:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a088. 00000100:00000200:2.0:1790701723.535885:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ea00 x1877686537440384/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1790701723.536351:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074e680. 02000000:00000010:3.0:1790701723.536361:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441f00. 02000000:00000010:2.0:1790701723.536404:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928200. 00000400:00000200:2.0:1790701723.536410:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701723.536413:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2478. 00000400:00000010:2.0:1790701723.536417:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a088. 00000100:00000200:2.0:1790701723.536421:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440512, portal 10 00000400:00000010:2.0:1790701723.536425:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a220. 00000100:00000200:2.0:1790701723.536434:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537440512, offset 0 00000400:00000200:2.0:1790701723.536439:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701723.536443:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701723.536448:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701723.536452:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701723.536454:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701723.536457:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701723.536460:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701723.536463:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [159:159:256:4294967295] 192.168.204.157@tcp LPNI seq info [159:159:8:4294967295] 00000400:00000200:2.0:1790701723.536467:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701723.536474:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701723.536479:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701723.536483:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4400. 00000800:00000200:2.0:1790701723.536487:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701723.536492:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701723.536495:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701723.536509:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701723.536612:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701723.536615:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4400. 00000400:00000200:3.0:1790701723.536619:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701723.536623:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701723.536627:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a220 00000400:00000010:3.0:1790701723.536628:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a220. 00000100:00000200:3.0:1790701723.536635:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074e680 x1877686537440512/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701723.538576:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538587:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701723.538596:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538685:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538694:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538698:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104c80 00000400:00000200:3.0:1790701723.538704:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538710:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x925 [1] + 224 00000400:00000200:3.0:1790701723.538722:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701723.538736:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701723.538751:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ea00 x1877686537440384/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701723.538792:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538796:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701723.538801:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538804:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538806:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538809:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104c00 00000400:00000200:3.0:1790701723.538812:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.538964:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x915 [1] + 224 00000400:00000200:3.0:1790701723.538977:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701723.538981:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701723.538987:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074fb80 x1877686537440256/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701723.539021:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2580. 00000400:00000200:3.0:1790701723.539027:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a2a8 00000400:00000010:3.0:1790701723.539029:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a2a8. 00000100:00000200:3.0:1790701723.539037:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ea00 x1877686537440384/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701723.539046:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ea00 x1877686537440384/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701723.539079:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2dc0. 00000400:00000200:3.0:1790701723.539083:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a330 00000400:00000010:3.0:1790701723.539084:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a330. 00000100:00000200:3.0:1790701723.539089:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074fb80 x1877686537440256/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701723.539097:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074fb80 x1877686537440256/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701723.539112:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928600. 02000000:00000010:3.0:1790701723.539117:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441800. 00000100:00000010:3.0:1790701723.539128:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ea00. 02000000:00000010:3.0:1790701723.539132:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928000. 02000000:00000010:3.0:1790701723.539135:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441200. 00000100:00000010:3.0:1790701723.539137:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074fb80. 00000400:00000200:3.0:1790701723.541488:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.541499:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701723.541507:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.541511:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.541517:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.541521:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104d00 00000400:00000200:3.0:1790701723.541526:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701723.541532:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x935 [1] + 224 00000400:00000200:3.0:1790701723.541546:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701723.541550:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701723.541566:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074e680 x1877686537440512/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701723.542000:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2478. 00000400:00000200:3.0:1790701723.542006:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a088 00000400:00000010:3.0:1790701723.542008:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a088. 00000100:00000200:3.0:1790701723.542016:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074e680 x1877686537440512/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701723.542025:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074e680 x1877686537440512/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701739 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701723.542058:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928200. 02000000:00000010:3.0:1790701723.542063:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441f00. 00000100:00000010:3.0:1790701723.542068:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074e680. 00000100:00000010:2.0:1790701724.817061:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc9edb420. 00000100:00000010:3.0:1790701724.819102:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc9edb240. 00000001:02000400:0.0:1790701727.314992:0:137143:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 612: lnet_legacy2yaml: malformed IP pattern ========================================================== 13:08:45 (1790701725) 00000100:00000010:3.0:1790701728.652660:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074e680. 02000000:00000010:3.0:1790701728.652690:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441700. 00000100:00000010:3.0:1790701728.652735:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074fb80. 02000000:00000010:3.0:1790701728.652749:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441a00. 00000100:00000010:3.0:1790701728.652758:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ea00. 02000000:00000010:3.0:1790701728.652762:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441900. 02000000:00000010:3.0:1790701728.652797:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bce00. 00000400:00000200:3.0:1790701728.652808:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701728.652813:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223840. 00000400:00000010:3.0:1790701728.652843:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060990. 00000100:00000200:3.0:1790701728.652851:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440768, portal 10 00000400:00000010:3.0:1790701728.652859:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060330. 00000100:00000200:3.0:1790701728.652861:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537440768, offset 0 00000400:00000200:3.0:1790701728.652871:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701728.652878:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.652886:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701728.652892:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701728.652898:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701728.652902:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701728.652905:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701728.652909:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [160:160:256:4294967295] 192.168.204.157@tcp LPNI seq info [160:160:8:4294967295] 00000400:00000200:3.0:1790701728.652914:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.652924:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701728.652930:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701728.652936:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441100. 00000800:00000200:3.0:1790701728.652943:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701728.652952:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701728.652957:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701728.652973:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701728.652979:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4c60. 00000100:00000010:3.0:1790701728.652989:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4000. 02000000:00000010:3.0:1790701728.653014:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bd600. 00000400:00000200:3.0:1790701728.653017:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701728.653020:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62234d0. 00000400:00000010:3.0:1790701728.653029:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701728.653032:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440640, portal 25 00000400:00000010:3.0:1790701728.653036:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060000. 00000100:00000200:3.0:1790701728.653038:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537440640, offset 0 00000400:00000200:3.0:1790701728.653043:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701728.653047:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.653050:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701728.653053:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701728.653055:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701728.653057:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701728.653060:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701728.653062:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [161:161:255:4294967295] 192.168.204.157@tcp LPNI seq info [161:161:7:4294967295] 00000400:00000200:3.0:1790701728.653066:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.653072:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701728.653076:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701728.653080:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441b00. 00000800:00000200:3.0:1790701728.653084:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701728.653088:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701728.653090:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701728.653095:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701728.653107:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc83bc400. 00000400:00000200:3.0:1790701728.653111:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701728.653114:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223aa8. 00000400:00000010:3.0:1790701728.653117:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060b28. 00000100:00000200:3.0:1790701728.653119:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537440896, portal 10 00000400:00000010:3.0:1790701728.653123:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060550. 00000100:00000200:3.0:1790701728.653126:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537440896, offset 0 00000400:00000200:3.0:1790701728.653130:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701728.653134:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.653136:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701728.653138:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701728.653147:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701728.653150:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701728.653151:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701728.653154:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [162:162:254:4294967295] 192.168.204.157@tcp LPNI seq info [162:162:6:4294967295] 00000400:00000200:3.0:1790701728.653157:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.653162:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701728.653167:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701728.653170:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc93b4100. 00000800:00000200:3.0:1790701728.653174:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701728.653178:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701728.653180:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc93b4100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701728.653186:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701728.653233:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701728.653237:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441100. 00000400:00000200:3.0:1790701728.653243:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701728.653247:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701728.655659:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060330 00000400:00000010:3.0:1790701728.655671:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060330. 00000100:00000200:3.0:1790701728.655693:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074fb80 x1877686537440768/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701728.655739:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701728.655743:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441b00. 00000400:00000200:3.0:1790701728.655748:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701728.655753:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701728.655758:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060000 00000400:00000010:3.0:1790701728.655760:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060000. 00000100:00000200:3.0:1790701728.655766:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074e680 x1877686537440640/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701728.655857:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701728.655861:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc93b4100. 00000400:00000200:3.0:1790701728.655864:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701728.655876:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701728.655881:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060550 00000400:00000010:3.0:1790701728.655883:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060550. 00000100:00000200:3.0:1790701728.655889:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ea00 x1877686537440896/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701728.659866:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.659879:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701728.659889:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.659896:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.659903:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.659907:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104d80 00000400:00000200:3.0:1790701728.659913:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.659919:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x955 [1] + 224 00000400:00000200:3.0:1790701728.659934:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701728.659940:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701728.659956:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074e680 x1877686537440640/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701728.660006:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62234d0. 00000400:00000200:3.0:1790701728.660012:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060198 00000400:00000010:3.0:1790701728.660014:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701728.660020:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074e680 x1877686537440640/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701728.660028:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074e680 x1877686537440640/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701728.660060:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bd600. 02000000:00000010:3.0:1790701728.660066:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441700. 00000100:00000010:3.0:1790701728.660072:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074e680. 00000400:00000200:3.0:1790701728.660617:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.660623:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701728.660629:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.660647:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.660651:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.660655:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104e00 00000400:00000200:3.0:1790701728.660659:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.660666:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x945 [1] + 224 00000400:00000200:3.0:1790701728.660678:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701728.660683:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701728.660691:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074fb80 x1877686537440768/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701728.661349:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223840. 00000400:00000200:2.0:1790701728.661356:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060990 00000400:00000010:2.0:1790701728.661358:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060990. 00000100:00000200:2.0:1790701728.661367:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074fb80 x1877686537440768/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701728.661377:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074fb80 x1877686537440768/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701728.661401:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bce00. 02000000:00000010:2.0:1790701728.661406:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441a00. 00000100:00000010:2.0:1790701728.661409:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074fb80. 00000400:00000200:3.0:1790701728.663627:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.663651:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701728.663660:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.663666:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.663672:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.663676:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104e80 00000400:00000200:3.0:1790701728.663682:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701728.663689:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x965 [1] + 224 00000400:00000200:3.0:1790701728.663705:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701728.663710:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701728.663725:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ea00 x1877686537440896/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701728.663777:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223aa8. 00000400:00000200:3.0:1790701728.663783:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060b28 00000400:00000010:3.0:1790701728.663785:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060b28. 00000100:00000200:3.0:1790701728.663791:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ea00 x1877686537440896/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701728.663800:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ea00 x1877686537440896/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701744 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701728.663861:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc83bc400. 02000000:00000010:3.0:1790701728.663880:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441900. 00000100:00000010:3.0:1790701728.663886:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ea00. 00000100:00000010:3.0:1790701729.931958:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4000. 00000100:00000010:2.0:1790701729.932886:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4c60. 00000100:00000010:1.0:1790701733.776804:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701733.777976:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5b00. 00000100:00000010:1.0:1790701733.778023:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701733.778029:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5800. 00000100:00000010:1.0:1790701733.778037:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701733.778042:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5900. 02000000:00000010:1.0:1790701733.778085:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce1366e00. 00000400:00000200:1.0:1790701733.778098:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701733.778103:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1855d8. 00000400:00000010:1.0:1790701733.778111:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701733.778120:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441024, portal 25 00000400:00000010:1.0:1790701733.778134:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2ee0. 00000100:00000200:1.0:1790701733.778138:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537441024, offset 0 00000400:00000200:1.0:1790701733.778147:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701733.778156:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701733.778167:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701733.778173:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701733.778178:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701733.778183:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701733.778187:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701733.778190:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [163:163:256:4294967295] 192.168.204.157@tcp LPNI seq info [163:163:8:4294967295] 00000400:00000200:1.0:1790701733.778195:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701733.778206:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701733.778212:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701733.778218:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5700. 00000800:00000200:1.0:1790701733.778226:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701733.778235:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701733.778240:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701733.778263:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701733.778269:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd480. 00000100:00000010:1.0:1790701733.778280:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd600. 02000000:00000010:1.0:1790701733.778293:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea1000. 00000400:00000200:1.0:1790701733.778297:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701733.778300:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1857e8. 00000400:00000010:1.0:1790701733.778304:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b23b8. 00000100:00000200:1.0:1790701733.778307:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441280, portal 10 00000400:00000010:1.0:1790701733.778312:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2e58. 00000100:00000200:1.0:1790701733.778314:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537441280, offset 0 00000400:00000200:1.0:1790701733.778318:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701733.778321:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701733.778324:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701733.778327:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701733.778335:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701733.778337:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701733.778339:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701733.778342:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [164:164:255:4294967295] 192.168.204.157@tcp LPNI seq info [164:164:7:4294967295] 00000400:00000200:1.0:1790701733.778345:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701733.778351:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701733.778355:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701733.778358:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5000. 00000800:00000200:1.0:1790701733.778362:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701733.778366:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701733.778368:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701733.778374:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701733.778395:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bd400. 00000400:00000200:1.0:1790701733.778399:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701733.778401:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185ec8. 00000400:00000010:1.0:1790701733.778405:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2bb0. 00000100:00000200:1.0:1790701733.778408:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441152, portal 10 00000400:00000010:1.0:1790701733.778412:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2088. 00000100:00000200:1.0:1790701733.778414:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537441152, offset 0 00000400:00000200:1.0:1790701733.778419:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701733.778422:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701733.778425:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701733.778427:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701733.778429:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701733.778431:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701733.778433:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701733.778435:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [165:165:254:4294967295] 192.168.204.157@tcp LPNI seq info [165:165:6:4294967295] 00000400:00000200:1.0:1790701733.778439:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701733.778443:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701733.778447:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701733.778467:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5200. 00000800:00000200:3.0:1790701733.778660:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701733.778665:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5700. 00000400:00000200:3.0:1790701733.778671:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701733.778676:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701733.778682:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2ee0 00000400:00000010:3.0:1790701733.778685:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2ee0. 00000100:00000200:3.0:1790701733.778696:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537441024/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701733.778777:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701733.778780:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5000. 00000400:00000200:3.0:1790701733.778783:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701733.778786:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701733.778789:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2e58 00000400:00000010:3.0:1790701733.778790:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2e58. 00000100:00000200:3.0:1790701733.778795:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537441280/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701733.779653:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701733.779660:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701733.779665:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701733.779687:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701733.779807:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701733.779811:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5200. 00000400:00000200:3.0:1790701733.779815:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701733.779846:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701733.779851:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2088 00000400:00000010:3.0:1790701733.779853:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2088. 00000100:00000200:3.0:1790701733.779860:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537441152/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701733.780854:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.780860:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701733.780867:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.780871:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.780887:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.780891:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7104f00 00000400:00000200:3.0:1790701733.780896:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.780901:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x975 [1] + 224 00000400:00000200:3.0:1790701733.780915:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701733.780919:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701733.780929:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537441024/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701733.781293:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1855d8. 00000400:00000200:1.0:1790701733.781300:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2990 00000400:00000010:1.0:1790701733.781302:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701733.781310:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537441024/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701733.781320:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537441024/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701733.781355:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce1366e00. 02000000:00000010:1.0:1790701733.781365:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5b00. 00000100:00000010:1.0:1790701733.781372:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000400:00000200:3.0:1790701733.782079:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782084:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701733.782091:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782095:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782099:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782103:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105000 00000400:00000200:3.0:1790701733.782107:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782111:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x97d [1] + 224 00000400:00000200:3.0:1790701733.782121:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701733.782124:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701733.782133:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537441280/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701733.782214:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782218:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701733.782221:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782224:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782227:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782268:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7104f80 00000400:00000200:3.0:1790701733.782273:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701733.782278:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x985 [1] + 224 00000400:00000200:3.0:1790701733.782288:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701733.782298:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701733.782304:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537441152/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701733.792830:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1857e8. 00000400:00000200:1.0:1790701733.792841:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b23b8 00000400:00000010:1.0:1790701733.792844:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b23b8. 00000100:00000200:1.0:1790701733.792862:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537441280/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701733.792875:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537441280/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701733.792914:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea1000. 02000000:00000010:1.0:1790701733.792921:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5900. 00000100:00000010:1.0:1790701733.792926:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000010:1.0:1790701733.792936:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185ec8. 00000400:00000200:1.0:1790701733.792938:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2bb0 00000400:00000010:1.0:1790701733.792939:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2bb0. 00000100:00000200:1.0:1790701733.792945:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537441152/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701733.793175:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537441152/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701749 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701733.793194:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bd400. 02000000:00000010:1.0:1790701733.793198:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5800. 00000100:00000010:1.0:1790701733.793201:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000100:00000010:3.0:1790701735.115971:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd600. 00000100:00000010:2.0:1790701735.116627:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd480. 00000001:02000400:0.0:1790701736.346071:0:137722:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 613: lnet_legacy2yaml: multiple IP patterns in single rule ========================================================== 13:08:54 (1790701734) 00000100:00000010:1.0:1790701738.892332:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701738.892364:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5900. 00000100:00000010:1.0:1790701738.892397:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701738.892402:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5600. 00000100:00000010:1.0:1790701738.892563:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701738.892569:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5400. 02000000:00000010:1.0:1790701738.892603:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acd000. 00000400:00000200:1.0:1790701738.892613:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701738.892617:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185ec8. 00000400:00000010:1.0:1790701738.892624:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2bb0. 00000100:00000200:1.0:1790701738.892631:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441408, portal 25 00000400:00000010:1.0:1790701738.892637:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b23b8. 00000100:00000200:1.0:1790701738.892641:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537441408, offset 0 00000400:00000200:1.0:1790701738.892652:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701738.892661:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701738.892669:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701738.892674:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701738.892679:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701738.892684:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701738.892687:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701738.892690:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [166:166:256:4294967295] 192.168.204.157@tcp LPNI seq info [166:166:8:4294967295] 00000400:00000200:1.0:1790701738.892695:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701738.892706:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701738.892713:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701738.892725:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5500. 00000800:00000200:1.0:1790701738.892733:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701738.892743:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701738.892748:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701738.892760:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701738.892765:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd240. 00000100:00000010:1.0:1790701738.892777:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd180. 02000000:00000010:1.0:1790701738.892789:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acc400. 00000400:00000200:1.0:1790701738.892793:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701738.892795:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1857e8. 00000400:00000010:1.0:1790701738.892799:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701738.892802:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441664, portal 10 00000400:00000010:1.0:1790701738.892808:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2f68. 00000100:00000200:1.0:1790701738.892811:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537441664, offset 0 00000400:00000200:1.0:1790701738.892815:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701738.892845:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701738.892850:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701738.892852:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701738.892855:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701738.892857:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701738.892860:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701738.892862:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [167:167:255:4294967295] 192.168.204.157@tcp LPNI seq info [167:167:7:4294967295] 00000400:00000200:1.0:1790701738.892866:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701738.892872:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701738.893256:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701738.893262:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5500. 00000400:00000200:3.0:1790701738.893269:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701738.893275:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701738.893280:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b23b8 00000400:00000010:3.0:1790701738.893282:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b23b8. 00000100:00000200:3.0:1790701738.893292:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537441408/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1790701738.893393:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701738.893398:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5c00. 00000800:00000200:1.0:1790701738.893402:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 02000000:00000010:0.0:1790701738.893669:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce12ae600. 00000400:00000200:0.0:1790701738.893675:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701738.893679:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f370. 00000400:00000010:0.0:1790701738.893683:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701738.893687:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441536, portal 10 00000400:00000010:0.0:1790701738.893694:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb660. 00000100:00000200:0.0:1790701738.893696:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537441536, offset 0 00000400:00000200:0.0:1790701738.893702:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701738.893706:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701738.893711:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701738.893714:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701738.893717:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701738.893720:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701738.893722:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701738.893725:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [168:168:255:4294967295] 192.168.204.157@tcp LPNI seq info [168:168:7:4294967295] 00000400:00000200:0.0:1790701738.893736:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701738.893744:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701738.893749:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701738.893753:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285300. 00000800:00000200:0.0:1790701738.893757:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701738.893762:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701738.893766:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701738.893779:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:1.0:1790701738.893977:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701738.893980:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701738.893988:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701738.894172:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701738.894176:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285300. 00000400:00000200:3.0:1790701738.894179:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701738.894184:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701738.894188:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb660 00000400:00000010:3.0:1790701738.894190:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb660. 00000100:00000200:3.0:1790701738.894196:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537441536/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701738.894242:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701738.894245:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5c00. 00000400:00000200:3.0:1790701738.894249:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701738.894252:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701738.894254:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2f68 00000400:00000010:3.0:1790701738.894256:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2f68. 00000100:00000200:3.0:1790701738.894261:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537441664/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701738.897113:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.897125:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701738.897136:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.897148:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.897154:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.897158:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105080 00000400:00000200:3.0:1790701738.897164:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.897170:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x98d [1] + 224 00000400:00000200:3.0:1790701738.897182:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701738.897187:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701738.897204:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537441408/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701738.898384:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.898390:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701738.898397:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.898401:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.898406:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.898409:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105100 00000400:00000200:3.0:1790701738.898413:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.898418:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x99d [1] + 224 00000400:00000200:3.0:1790701738.898427:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701738.898431:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701738.898967:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537441536/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701738.899219:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f370. 00000400:00000200:0.0:1790701738.899227:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbcc0 00000400:00000010:0.0:1790701738.899230:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701738.899239:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537441536/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701738.899249:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537441536/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701738.899290:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce12ae600. 02000000:00000010:0.0:1790701738.899299:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5600. 00000100:00000010:0.0:1790701738.899305:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000010:1.0:1790701738.899424:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185ec8. 00000400:00000200:1.0:1790701738.899429:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2bb0 00000400:00000010:1.0:1790701738.899431:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2bb0. 00000100:00000200:1.0:1790701738.899437:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537441408/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701738.899446:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537441408/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701738.899459:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acd000. 02000000:00000010:1.0:1790701738.899462:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5900. 00000100:00000010:1.0:1790701738.899465:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000400:00000200:3.0:1790701738.900417:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.900424:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701738.900431:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.900436:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.900516:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.900590:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105180 00000400:00000200:3.0:1790701738.900595:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701738.900601:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x995 [1] + 224 00000400:00000200:3.0:1790701738.900619:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701738.900837:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701738.900852:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537441664/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701738.901612:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1857e8. 00000400:00000200:1.0:1790701738.901620:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2990 00000400:00000010:1.0:1790701738.901623:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701738.901631:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537441664/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701738.901641:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537441664/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701754 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701738.901673:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acc400. 02000000:00000010:1.0:1790701738.901680:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5400. 00000100:00000010:1.0:1790701738.901685:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000100:00000010:2.0:1790701740.172092:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd240. 00000100:00000010:3.0:1790701740.172950:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd180. 00000100:00000010:1.0:1790701744.013174:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701744.013210:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5800. 00000100:00000010:1.0:1790701744.013266:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701744.013272:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5400. 00000100:00000010:1.0:1790701744.013281:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f6300. 02000000:00000010:1.0:1790701744.013285:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5900. 02000000:00000010:1.0:1790701744.013393:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea0600. 00000400:00000200:1.0:1790701744.013407:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701744.013413:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1857e8. 00000400:00000010:1.0:1790701744.013424:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2990. 00000100:00000200:1.0:1790701744.013446:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441920, portal 10 00000400:00000010:1.0:1790701744.013455:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2bb0. 00000100:00000200:1.0:1790701744.013460:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537441920, offset 0 00000400:00000200:1.0:1790701744.013474:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701744.013485:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701744.013495:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701744.013501:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701744.013507:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701744.013511:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701744.013514:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701744.013519:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [169:169:256:4294967295] 192.168.204.157@tcp LPNI seq info [169:169:8:4294967295] 00000400:00000200:1.0:1790701744.013525:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701744.013539:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701744.013546:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701744.013551:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5d00. 00000800:00000200:1.0:1790701744.013559:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701744.013567:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701744.013573:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701744.013596:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701744.013603:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd780. 00000100:00000010:1.0:1790701744.013616:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cdba0. 02000000:00000010:1.0:1790701744.013643:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ce0ea1400. 00000400:00000200:1.0:1790701744.013647:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701744.013650:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185ec8. 00000400:00000010:1.0:1790701744.013654:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2110. 00000100:00000200:1.0:1790701744.013657:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537441792, portal 25 00000400:00000010:1.0:1790701744.013660:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2d48. 00000100:00000200:1.0:1790701744.013663:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537441792, offset 0 00000400:00000200:1.0:1790701744.013667:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701744.013671:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701744.013675:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701744.013683:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701744.013686:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701744.013689:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701744.013691:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701744.013695:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [170:170:255:4294967295] 192.168.204.157@tcp LPNI seq info [170:170:7:4294967295] 00000400:00000200:1.0:1790701744.013699:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701744.013705:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701744.013710:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701744.013713:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5100. 00000800:00000200:1.0:1790701744.013717:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701744.013721:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701744.013724:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701744.013729:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701744.013744:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc3b28a00. 00000400:00000200:1.0:1790701744.013748:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701744.013751:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1855d8. 00000400:00000010:1.0:1790701744.013754:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2770. 00000100:00000200:1.0:1790701744.013757:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442048, portal 10 00000400:00000010:1.0:1790701744.013762:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b27f8. 00000100:00000200:1.0:1790701744.013764:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537442048, offset 0 00000400:00000200:1.0:1790701744.013768:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701744.013772:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701744.013775:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701744.013777:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701744.013780:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701744.013782:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701744.013784:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701744.013787:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [171:171:254:4294967295] 192.168.204.157@tcp LPNI seq info [171:171:6:4294967295] 00000400:00000200:1.0:1790701744.013791:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701744.013796:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701744.013809:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701744.013812:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5f00. 00000800:00000200:1.0:1790701744.013816:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701744.013845:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701744.013848:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701744.013854:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:2.0:1790701744.015086:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701744.015095:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5d00. 00000400:00000200:2.0:1790701744.015104:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701744.015113:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701744.015120:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2bb0 00000400:00000010:2.0:1790701744.015123:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2bb0. 00000100:00000200:2.0:1790701744.015135:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537441920/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701744.015161:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701744.015164:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5100. 00000400:00000200:2.0:1790701744.015167:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701744.015171:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701744.015174:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2d48 00000400:00000010:2.0:1790701744.015176:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2d48. 00000100:00000200:2.0:1790701744.015183:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537441792/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1790701744.015271:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1790701744.015274:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5f00. 00000400:00000200:2.0:1790701744.015276:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701744.015280:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:2.0:1790701744.015283:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b27f8 00000400:00000010:2.0:1790701744.015284:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b27f8. 00000100:00000200:2.0:1790701744.015289:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f6300 x1877686537442048/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701744.021797:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021813:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701744.021841:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021847:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021853:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021858:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105280 00000400:00000200:3.0:1790701744.021865:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021872:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x9a5 [1] + 224 00000400:00000200:3.0:1790701744.021887:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701744.021892:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701744.021921:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537441920/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701744.021967:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021970:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701744.021974:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021977:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021981:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021984:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105300 00000400:00000200:3.0:1790701744.021988:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.021993:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x9b5 [1] + 224 00000400:00000200:3.0:1790701744.021999:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701744.022003:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000010:0.0:1790701744.022013:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1857e8. 00000400:00000200:0.0:1790701744.022022:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2990 00000400:00000010:0.0:1790701744.022031:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2990. 00000100:00000200:0.0:1790701744.022041:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537441920/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701744.022052:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537441920/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701744.022094:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea0600. 02000000:00000010:0.0:1790701744.022103:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5400. 00000100:00000010:0.0:1790701744.022111:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 00000100:00000200:3.0:1790701744.025198:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f6300 x1877686537442048/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701744.025285:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701744.025314:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1855d8. 00000400:00000200:3.0:1790701744.025316:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701744.025328:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701744.025331:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2770 00000400:00000010:0.0:1790701744.025335:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2770. 00000400:00000200:3.0:1790701744.025336:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000200:0.0:1790701744.025345:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f6300 x1877686537442048/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701744.025355:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f6300 x1877686537442048/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701744.025397:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc3b28a00. 02000000:00000010:0.0:1790701744.025405:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5900. 00000100:00000010:0.0:1790701744.025411:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f6300. 00000400:00000200:3.0:1790701744.025426:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.025431:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105200 00000400:00000200:3.0:1790701744.025437:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701744.025444:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x9ad [1] + 224 00000400:00000200:3.0:1790701744.025464:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701744.025469:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701744.025479:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537441792/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701744.025645:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185ec8. 00000400:00000200:0.0:1790701744.025650:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2110 00000400:00000010:0.0:1790701744.025652:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2110. 00000100:00000200:0.0:1790701744.025657:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537441792/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701744.025666:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537441792/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701760 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701744.025681:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ce0ea1400. 02000000:00000010:0.0:1790701744.025685:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5800. 00000100:00000010:0.0:1790701744.025688:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 00000100:00000010:3.0:1790701745.292587:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cdba0. 00000100:00000010:2.0:1790701745.296451:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cfe2cd780. 00000001:02000400:2.0:1790701745.544810:0:138303:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 614: lnet_legacy2yaml: interfaces enumeration with multiple IP ranges ========================================================== 13:09:03 (1790701743) 00000100:00000010:3.0:1790701749.135260:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ea00. 02000000:00000010:3.0:1790701749.135480:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f800. 00000100:00000010:3.0:1790701749.135515:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074e680. 02000000:00000010:3.0:1790701749.135521:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f000. 00000100:00000010:3.0:1790701749.135529:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074df80. 02000000:00000010:3.0:1790701749.135533:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f700. 02000000:00000010:3.0:1790701749.135565:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bd400. 00000400:00000200:3.0:1790701749.135576:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701749.135580:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223aa8. 00000400:00000010:3.0:1790701749.135587:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060b28. 00000100:00000200:3.0:1790701749.135596:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442176, portal 25 00000400:00000010:3.0:1790701749.135603:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701749.135606:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537442176, offset 0 00000400:00000200:3.0:1790701749.135618:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701749.135627:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.135635:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701749.135640:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701749.135646:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701749.135650:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701749.135667:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701749.135671:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [172:172:256:4294967295] 192.168.204.157@tcp LPNI seq info [172:172:8:4294967295] 00000400:00000200:3.0:1790701749.135676:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.135687:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701749.135693:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701749.135698:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24ff00. 00000800:00000200:3.0:1790701749.135705:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701749.135724:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701749.135728:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701749.135743:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701749.135749:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4de0. 00000100:00000010:3.0:1790701749.135759:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4240. 02000000:00000010:3.0:1790701749.135771:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bc400. 00000400:00000200:3.0:1790701749.135774:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701749.135777:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62234d0. 00000400:00000010:3.0:1790701749.135780:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060550. 00000100:00000200:3.0:1790701749.135783:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442432, portal 10 00000400:00000010:3.0:1790701749.135788:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060000. 00000100:00000200:3.0:1790701749.135791:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537442432, offset 0 00000400:00000200:3.0:1790701749.135795:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701749.135798:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.135801:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701749.135803:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701749.135805:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701749.135808:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701749.135810:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701749.135813:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [173:173:255:4294967295] 192.168.204.157@tcp LPNI seq info [173:173:7:4294967295] 00000400:00000200:3.0:1790701749.135940:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.135947:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701749.135952:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701749.135962:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24f600. 00000800:00000200:3.0:1790701749.135966:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701749.135970:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701749.135973:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24f600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701749.135980:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701749.136000:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cf52fdc00. 00000400:00000200:3.0:1790701749.136004:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701749.136006:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62237e8. 00000400:00000010:3.0:1790701749.136010:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060330. 00000100:00000200:3.0:1790701749.136012:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442304, portal 10 00000400:00000010:3.0:1790701749.136017:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060908. 00000100:00000200:3.0:1790701749.136019:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537442304, offset 0 00000400:00000200:3.0:1790701749.136023:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701749.136027:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.136030:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701749.136033:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701749.136035:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701749.136038:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701749.136040:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701749.136043:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [174:174:254:4294967295] 192.168.204.157@tcp LPNI seq info [174:174:6:4294967295] 00000400:00000200:3.0:1790701749.136046:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.136052:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701749.136056:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701749.136059:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24fb00. 00000800:00000200:3.0:1790701749.136063:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701749.136066:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701749.136069:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24fb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701749.136074:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701749.136122:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701749.136126:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24ff00. 00000400:00000200:3.0:1790701749.136131:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701749.136141:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701749.136147:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060198 00000400:00000010:3.0:1790701749.136149:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701749.136628:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ea00 x1877686537442176/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701749.136655:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701749.136657:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24f600. 00000400:00000200:3.0:1790701749.136660:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701749.136663:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701749.136666:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060000 00000400:00000010:3.0:1790701749.136668:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060000. 00000100:00000200:3.0:1790701749.136673:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074df80 x1877686537442432/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701749.136989:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701749.136992:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24fb00. 00000400:00000200:3.0:1790701749.136995:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701749.136999:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701749.137002:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060908 00000400:00000010:3.0:1790701749.137004:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060908. 00000100:00000200:3.0:1790701749.137008:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074e680 x1877686537442304/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701749.140041:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.140053:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701749.140062:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.140067:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.140073:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.140076:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105380 00000400:00000200:3.0:1790701749.140082:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701749.140089:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x9bd [1] + 224 00000400:00000200:3.0:1790701749.140116:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701749.140123:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701749.140149:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ea00 x1877686537442176/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701749.140212:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223aa8. 00000400:00000200:3.0:1790701749.140217:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060b28 00000400:00000010:3.0:1790701749.140220:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060b28. 00000100:00000200:3.0:1790701749.140226:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ea00 x1877686537442176/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701749.140235:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ea00 x1877686537442176/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701749.140268:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bd400. 02000000:00000010:3.0:1790701749.140276:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f800. 00000100:00000010:3.0:1790701749.140386:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ea00. 00000400:00000200:2.0:1790701749.159411:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159423:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701749.159431:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159443:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159450:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159453:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105480 00000400:00000200:2.0:1790701749.159459:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159466:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x9cd [1] + 224 00000400:00000200:2.0:1790701749.159480:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701749.159485:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701749.159503:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074df80 x1877686537442432/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701749.159538:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159542:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701749.159545:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159547:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159550:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159552:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105400 00000400:00000200:2.0:1790701749.159556:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701749.159559:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x9dd [1] + 224 00000400:00000200:2.0:1790701749.159568:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701749.159571:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701749.159577:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074e680 x1877686537442304/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701749.159611:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62237e8. 00000400:00000200:2.0:1790701749.159615:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060330 00000400:00000010:2.0:1790701749.159618:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060330. 00000100:00000200:2.0:1790701749.159624:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074e680 x1877686537442304/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701749.159632:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074e680 x1877686537442304/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701749.159669:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62234d0. 00000400:00000200:2.0:1790701749.159680:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060550 00000400:00000010:2.0:1790701749.159681:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060550. 00000100:00000200:2.0:1790701749.159687:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074df80 x1877686537442432/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701749.159695:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074df80 x1877686537442432/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701765 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701749.159729:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cf52fdc00. 02000000:00000010:2.0:1790701749.159737:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f000. 00000100:00000010:2.0:1790701749.159743:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074e680. 02000000:00000010:2.0:1790701749.159746:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bc400. 02000000:00000010:2.0:1790701749.159748:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f700. 00000100:00000010:2.0:1790701749.159750:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074df80. 00000100:00000010:2.0:1790701750.413766:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4de0. 00000100:00000010:3.0:1790701750.413924:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4240. 00000001:02000400:2.0:1790701753.052582:0:138884:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 615: lnet_legacy2yaml: modprobe configuration file input ========================================================== 13:09:11 (1790701751) 00000100:00000010:1.0:1790701754.255864:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5180. 02000000:00000010:1.0:1790701754.255888:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5300. 00000100:00000010:1.0:1790701754.255933:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7100. 02000000:00000010:1.0:1790701754.255939:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5900. 02000000:00000010:0.0:1790701754.255947:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929600. 00000100:00000010:1.0:1790701754.255949:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f6300. 02000000:00000010:1.0:1790701754.255956:0:38:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5400. 00000400:00000200:0.0:1790701754.255962:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701754.255969:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f370. 00000400:00000010:0.0:1790701754.255977:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701754.255984:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442560, portal 25 00000400:00000010:0.0:1790701754.255991:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbbb0. 00000100:00000200:0.0:1790701754.255995:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537442560, offset 0 00000400:00000200:0.0:1790701754.256006:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701754.256015:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701754.256023:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701754.256029:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701754.256033:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701754.256049:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701754.256062:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701754.256066:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [175:175:256:4294967295] 192.168.204.157@tcp LPNI seq info [175:175:8:4294967295] 00000400:00000200:0.0:1790701754.256071:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701754.256092:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701754.256099:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701754.256105:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285500. 00000800:00000200:0.0:1790701754.256113:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701754.256130:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701754.256135:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701754.256156:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:0.0:1790701754.256162:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772c9c0. 00000100:00000010:0.0:1790701754.256173:0:123345:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772c000. 02000000:00000010:0.0:1790701754.256194:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb929800. 00000400:00000200:0.0:1790701754.256199:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701754.256201:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f0b0. 00000400:00000010:0.0:1790701754.256210:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbaa0. 00000100:00000200:0.0:1790701754.256214:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442816, portal 10 00000400:00000010:0.0:1790701754.256219:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb3b8. 00000100:00000200:0.0:1790701754.256221:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537442816, offset 0 00000400:00000200:0.0:1790701754.256226:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701754.256229:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701754.256233:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701754.256235:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701754.256237:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701754.256239:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701754.256241:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701754.256244:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [176:176:255:4294967295] 192.168.204.157@tcp LPNI seq info [176:176:7:4294967295] 00000400:00000200:0.0:1790701754.256247:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701754.256253:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701754.256265:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701754.256268:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285b00. 00000800:00000200:0.0:1790701754.256272:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701754.256275:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701754.256278:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701754.256283:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:0.0:1790701754.256297:0:123345:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928c00. 00000400:00000200:0.0:1790701754.256300:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701754.256302:0:123345:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f058. 00000400:00000010:0.0:1790701754.256305:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbee0. 00000100:00000200:0.0:1790701754.256307:0:123345:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442688, portal 10 00000400:00000010:0.0:1790701754.256312:0:123345:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecba18. 00000100:00000200:0.0:1790701754.256314:0:123345:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537442688, offset 0 00000400:00000200:0.0:1790701754.256317:0:123345:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701754.256320:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701754.256323:0:123345:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701754.256325:0:123345:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701754.256328:0:123345:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701754.256330:0:123345:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701754.256331:0:123345:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701754.256334:0:123345:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [177:177:254:4294967295] 192.168.204.157@tcp LPNI seq info [177:177:6:4294967295] 00000400:00000200:0.0:1790701754.256337:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701754.256342:0:123345:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701754.256346:0:123345:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701754.256349:0:123345:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe285000. 00000800:00000200:0.0:1790701754.256352:0:123345:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701754.256357:0:123345:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701754.256359:0:123345:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe285000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701754.256363:0:123345:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701754.256602:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701754.256607:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285500. 00000400:00000200:3.0:1790701754.256613:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701754.256619:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701754.256624:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbbb0 00000400:00000010:3.0:1790701754.256626:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbbb0. 00000100:00000200:3.0:1790701754.256639:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5180 x1877686537442560/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701754.256661:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701754.256663:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285b00. 00000400:00000200:3.0:1790701754.256665:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701754.256668:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701754.256671:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb3b8 00000400:00000010:3.0:1790701754.256672:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb3b8. 00000100:00000200:3.0:1790701754.256677:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f6300 x1877686537442816/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701754.256743:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701754.256752:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe285000. 00000400:00000200:3.0:1790701754.256756:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701754.256760:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701754.256763:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecba18 00000400:00000010:3.0:1790701754.256764:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecba18. 00000100:00000200:3.0:1790701754.256769:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7100 x1877686537442688/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701754.269945:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.269957:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701754.269966:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.269972:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.269979:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.269983:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105600 00000400:00000200:3.0:1790701754.269989:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.269995:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x9f5 [1] + 224 00000400:00000200:3.0:1790701754.270015:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701754.270022:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701754.270041:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f6300 x1877686537442816/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701754.270081:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.270085:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701754.270100:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.270103:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.270106:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.270109:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105580 00000400:00000200:3.0:1790701754.270113:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.270116:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x9fd [1] + 224 00000400:00000010:0.0:1790701754.270491:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f0b0. 00000400:00000200:0.0:1790701754.270505:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbaa0 00000400:00000010:0.0:1790701754.270508:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbaa0. 00000100:00000200:0.0:1790701754.270515:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f6300 x1877686537442816/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701754.270524:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f6300 x1877686537442816/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701754.270561:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929800. 02000000:00000010:0.0:1790701754.270566:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5400. 00000100:00000010:0.0:1790701754.270573:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f6300. 00000400:00000200:3.0:1790701754.270885:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701754.270891:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701754.270898:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7100 x1877686537442688/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701754.271419:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.271424:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701754.271428:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.271431:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.271434:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.271436:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105500 00000400:00000200:3.0:1790701754.271440:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701754.271443:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x9ed [1] + 224 00000400:00000200:3.0:1790701754.271450:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701754.271463:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701754.271469:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5180 x1877686537442560/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701754.271577:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f058. 00000400:00000200:0.0:1790701754.271581:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbee0 00000400:00000010:0.0:1790701754.271583:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbee0. 00000100:00000200:0.0:1790701754.271602:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7100 x1877686537442688/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701754.271615:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7100 x1877686537442688/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701754.271635:0:123345:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f370. 00000400:00000200:0.0:1790701754.271639:0:123345:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbcc0 00000400:00000010:0.0:1790701754.271641:0:123345:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701754.271645:0:123345:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5180 x1877686537442560/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701754.271652:0:123345:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5180 x1877686537442560/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701770 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701754.271664:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928c00. 02000000:00000010:0.0:1790701754.271667:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5900. 00000100:00000010:0.0:1790701754.271671:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7100. 02000000:00000010:0.0:1790701754.271674:0:123345:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb929600. 02000000:00000010:0.0:1790701754.271677:0:123345:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5300. 00000100:00000010:0.0:1790701754.271679:0:123345:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5180. 00000100:00000010:3.0:1790701755.531915:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772c000. 00000100:00000010:2.0:1790701755.532246:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772c9c0. 00000100:00000010:3.0:1790701759.377117:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ea00. 02000000:00000010:3.0:1790701759.377149:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24ff00. 00000100:00000010:3.0:1790701759.377376:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074f480. 02000000:00000010:3.0:1790701759.377383:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24fb00. 00000100:00000010:3.0:1790701759.377393:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074e300. 02000000:00000010:3.0:1790701759.377397:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f600. 02000000:00000010:3.0:1790701759.377443:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e91400. 00000400:00000200:3.0:1790701759.377457:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701759.377463:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223aa8. 00000400:00000010:3.0:1790701759.377473:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060b28. 00000100:00000200:3.0:1790701759.377484:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537442944, portal 25 00000400:00000010:3.0:1790701759.377491:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060908. 00000100:00000200:3.0:1790701759.377494:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537442944, offset 0 00000400:00000200:3.0:1790701759.377507:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701759.377517:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.377528:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701759.377544:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701759.377551:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701759.377555:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701759.377559:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701759.377566:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [178:178:256:4294967295] 192.168.204.157@tcp LPNI seq info [178:178:8:4294967295] 00000400:00000200:3.0:1790701759.377571:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.377586:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701759.377595:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701759.377602:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24f800. 00000800:00000200:3.0:1790701759.377611:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701759.377622:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701759.377629:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24f800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701759.377647:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701759.377654:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e46c0. 00000100:00000010:3.0:1790701759.377669:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4000. 02000000:00000010:3.0:1790701759.377681:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e90c00. 00000400:00000200:3.0:1790701759.377685:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701759.377688:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223580. 00000400:00000010:3.0:1790701759.377692:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060000. 00000100:00000200:3.0:1790701759.377695:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443200, portal 10 00000400:00000010:3.0:1790701759.377702:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701759.377704:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537443200, offset 0 00000400:00000200:3.0:1790701759.377708:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701759.377712:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.377716:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701759.377719:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701759.377721:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701759.377724:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701759.377726:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701759.377728:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [179:179:255:4294967295] 192.168.204.157@tcp LPNI seq info [179:179:7:4294967295] 00000400:00000200:3.0:1790701759.377732:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.377745:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701759.377749:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701759.377752:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24f700. 00000800:00000200:3.0:1790701759.377756:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701759.377760:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701759.377763:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24f700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701759.377769:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:3.0:1790701759.377788:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7e91600. 00000400:00000200:3.0:1790701759.377792:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701759.377794:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223210. 00000400:00000010:3.0:1790701759.377798:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060c38. 00000100:00000200:3.0:1790701759.377800:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443072, portal 10 00000400:00000010:3.0:1790701759.377805:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0606e8. 00000100:00000200:3.0:1790701759.377807:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537443072, offset 0 00000400:00000200:3.0:1790701759.377811:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701759.377815:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.377887:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701759.377890:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701759.377893:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701759.377895:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701759.377898:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701759.377900:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [180:180:254:4294967295] 192.168.204.157@tcp LPNI seq info [180:180:6:4294967295] 00000400:00000200:3.0:1790701759.377904:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.377909:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701759.377913:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701759.377917:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24f000. 00000800:00000200:3.0:1790701759.377921:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701759.377925:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701759.377928:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24f000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701759.377933:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701759.378012:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701759.378017:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24f800. 00000400:00000200:3.0:1790701759.378024:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701759.378030:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701759.378037:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060908 00000400:00000010:3.0:1790701759.378040:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060908. 00000100:00000200:3.0:1790701759.378051:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ea00 x1877686537442944/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701759.378077:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701759.378079:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24f700. 00000400:00000200:3.0:1790701759.378082:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701759.378085:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701759.378087:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060198 00000400:00000010:3.0:1790701759.378089:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701759.378093:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074e300 x1877686537443200/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701759.378181:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701759.378184:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24f000. 00000400:00000200:3.0:1790701759.378198:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701759.378202:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701759.378204:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0606e8 00000400:00000010:3.0:1790701759.378206:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0606e8. 00000100:00000200:3.0:1790701759.378210:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074f480 x1877686537443072/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701759.393036:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.393051:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701759.394606:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.394615:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.394623:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.394631:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105780 00000400:00000200:2.0:1790701759.394638:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.394645:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa15 [1] + 224 00000400:00000200:2.0:1790701759.394671:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701759.394677:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701759.394695:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074e300 x1877686537443200/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701759.394738:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.395270:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701759.395288:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.395291:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.395295:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.395299:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105700 00000400:00000200:2.0:1790701759.395303:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701759.395307:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa25 [1] + 224 00000400:00000200:2.0:1790701759.395325:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701759.395330:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701759.395340:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074f480 x1877686537443072/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701759.395451:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223210. 00000400:00000200:2.0:1790701759.395456:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060c38 00000400:00000010:2.0:1790701759.395459:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060c38. 00000100:00000200:2.0:1790701759.395466:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074f480 x1877686537443072/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701759.395476:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074f480 x1877686537443072/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701759.396450:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.396459:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701759.396467:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.396472:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.396478:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.396482:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105680 00000400:00000200:3.0:1790701759.396487:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701759.396504:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0xa05 [1] + 224 00000400:00000200:3.0:1790701759.396517:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701759.396522:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701759.396534:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ea00 x1877686537442944/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701759.397257:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223aa8. 00000400:00000200:2.0:1790701759.397268:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060b28 00000400:00000010:2.0:1790701759.397271:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060b28. 00000100:00000200:2.0:1790701759.397280:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ea00 x1877686537442944/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701759.397291:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ea00 x1877686537442944/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1790701759.397314:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223580. 00000400:00000200:2.0:1790701759.397316:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060000 00000400:00000010:2.0:1790701759.397318:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060000. 00000100:00000200:2.0:1790701759.397323:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074e300 x1877686537443200/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701759.397352:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074e300 x1877686537443200/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701775 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701759.397376:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e91600. 02000000:00000010:2.0:1790701759.397382:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24fb00. 00000100:00000010:2.0:1790701759.397387:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074f480. 02000000:00000010:2.0:1790701759.397391:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e91400. 02000000:00000010:2.0:1790701759.397393:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24ff00. 00000100:00000010:2.0:1790701759.397395:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ea00. 02000000:00000010:2.0:1790701759.397398:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7e90c00. 02000000:00000010:2.0:1790701759.397402:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f600. 00000100:00000010:2.0:1790701759.397404:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074e300. 00000100:00000010:2.0:1790701760.717388:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e46c0. 00000100:00000010:3.0:1790701760.742331:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4000. 00000001:02000400:2.0:1790701763.455288:0:139466:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 616: lnet_legacy2yaml: combining modprobe file with explicit ip2nets file ========================================================== 13:09:21 (1790701761) 00000100:00000010:3.0:1790701764.494060:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074f100. 02000000:00000010:3.0:1790701764.494090:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f700. 00000100:00000010:3.0:1790701764.494134:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ed80. 02000000:00000010:3.0:1790701764.494139:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24fa00. 00000100:00000010:3.0:1790701764.494149:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074e300. 02000000:00000010:3.0:1790701764.494153:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24fd00. 02000000:00000010:3.0:1790701764.495560:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928600. 00000400:00000200:3.0:1790701764.495579:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701764.495585:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223dc0. 00000400:00000010:3.0:1790701764.495595:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0606e8. 00000100:00000200:3.0:1790701764.495605:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443328, portal 25 00000400:00000010:3.0:1790701764.495612:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701764.495615:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537443328, offset 0 00000400:00000200:3.0:1790701764.495626:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701764.495637:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.495648:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701764.495654:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701764.495659:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701764.495663:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701764.495666:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701764.495670:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [181:181:256:4294967295] 192.168.204.157@tcp LPNI seq info [181:181:8:4294967295] 00000400:00000200:3.0:1790701764.495675:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.495686:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701764.495694:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701764.495718:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24f600. 00000800:00000200:3.0:1790701764.495726:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:3.0:1790701764.495736:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701764.495742:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24f600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701764.495768:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:3.0:1790701764.495775:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e46c0. 00000100:00000010:3.0:1790701764.495788:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8ce18e4840. 02000000:00000010:3.0:1790701764.495803:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8ccb928400. 00000400:00000200:3.0:1790701764.495808:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:3.0:1790701764.495811:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223a50. 00000400:00000010:3.0:1790701764.495815:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060908. 00000100:00000200:3.0:1790701764.495835:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443584, portal 10 00000400:00000010:3.0:1790701764.495842:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060660. 00000100:00000200:3.0:1790701764.495845:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537443584, offset 0 00000400:00000200:3.0:1790701764.495850:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:3.0:1790701764.495866:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.495869:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:3.0:1790701764.495877:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1790701764.495880:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:3.0:1790701764.497785:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1790701764.497792:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:3.0:1790701764.497797:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [182:182:255:4294967295] 192.168.204.157@tcp LPNI seq info [182:182:7:4294967295] 00000400:00000200:3.0:1790701764.497806:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.497832:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:3.0:1790701764.497843:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:3.0:1790701764.497865:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe24ff00. 00000800:00000200:3.0:1790701764.497874:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 02000000:00000010:2.0:1790701764.498097:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc4d1cc00. 00000400:00000200:2.0:1790701764.498106:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:2.0:1790701764.498111:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf60b2aa8. 00000400:00000010:2.0:1790701764.498119:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6a880. 00000100:00000200:2.0:1790701764.498128:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443456, portal 10 00000400:00000010:2.0:1790701764.498136:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8ccbb6ae58. 00000100:00000200:2.0:1790701764.498139:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537443456, offset 0 00000400:00000200:2.0:1790701764.498147:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:2.0:1790701764.498156:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701764.498165:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:2.0:1790701764.498170:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1790701764.498174:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:2.0:1790701764.498177:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1790701764.498179:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:2.0:1790701764.498182:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [183:183:254:4294967295] 192.168.204.157@tcp LPNI seq info [183:183:6:4294967295] 00000400:00000200:2.0:1790701764.498187:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701764.498215:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:2.0:1790701764.498220:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:2.0:1790701764.498225:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56f4c00. 00000800:00000200:2.0:1790701764.498229:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:2.0:1790701764.498234:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:2.0:1790701764.498240:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56f4c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1790701764.498249:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701764.498269:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:3.0:1790701764.498272:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe24ff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1790701764.498278:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701764.498504:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701764.498509:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24f600. 00000400:00000200:3.0:1790701764.498519:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701764.498524:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701764.498532:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060198 00000400:00000010:3.0:1790701764.498535:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060198. 00000100:00000200:3.0:1790701764.498546:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074f100 x1877686537443328/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701764.498574:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701764.498577:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56f4c00. 00000400:00000200:3.0:1790701764.498580:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701764.498583:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701764.498586:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6ae58 00000400:00000010:3.0:1790701764.498587:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6ae58. 00000100:00000200:3.0:1790701764.498594:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074ed80 x1877686537443456/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701764.499614:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701764.499622:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe24ff00. 00000400:00000200:3.0:1790701764.499629:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701764.499637:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701764.499642:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060660 00000400:00000010:3.0:1790701764.499645:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060660. 00000100:00000200:3.0:1790701764.499654:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd074e300 x1877686537443584/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701764.503771:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.503785:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701764.503796:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.503802:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.503808:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.503812:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105880 00000400:00000200:3.0:1790701764.503842:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.503881:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa55 [1] + 224 00000400:00000200:3.0:1790701764.507503:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701764.507515:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701764.507537:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074ed80 x1877686537443456/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701764.507590:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507595:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701764.507601:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507606:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507611:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507615:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105900 00000400:00000200:3.0:1790701764.507619:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507625:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa45 [1] + 224 00000400:00000200:3.0:1790701764.507631:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701764.507634:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701764.507640:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074e300 x1877686537443584/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701764.507665:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507668:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701764.507672:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507673:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507676:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507678:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105800 00000400:00000200:3.0:1790701764.507682:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701764.507686:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0xa35 [1] + 224 00000400:00000200:3.0:1790701764.507694:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701764.507697:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701764.507703:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd074f100 x1877686537443328/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701764.507778:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223dc0. 00000400:00000200:3.0:1790701764.507784:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0606e8 00000400:00000010:3.0:1790701764.507787:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0606e8. 00000100:00000200:3.0:1790701764.507794:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074f100 x1877686537443328/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701764.507803:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074f100 x1877686537443328/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1790701764.507865:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223a50. 00000400:00000200:3.0:1790701764.507868:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060908 00000400:00000010:3.0:1790701764.507870:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060908. 00000100:00000200:3.0:1790701764.507874:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074e300 x1877686537443584/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1790701764.507882:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074e300 x1877686537443584/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1790701764.507907:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928600. 02000000:00000010:3.0:1790701764.507913:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f700. 00000100:00000010:3.0:1790701764.507920:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074f100. 02000000:00000010:3.0:1790701764.507925:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8ccb928400. 02000000:00000010:3.0:1790701764.507927:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24fd00. 00000100:00000010:3.0:1790701764.507929:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074e300. 00000400:00000010:2.0:1790701764.508132:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf60b2aa8. 00000400:00000200:2.0:1790701764.508137:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8ccbb6a880 00000400:00000010:2.0:1790701764.508138:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8ccbb6a880. 00000100:00000200:2.0:1790701764.508147:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd074ed80 x1877686537443456/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701764.508155:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd074ed80 x1877686537443456/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701780 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1790701764.508173:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc4d1cc00. 02000000:00000010:2.0:1790701764.508177:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24fa00. 00000100:00000010:2.0:1790701764.508180:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ed80. 00000100:00000010:2.0:1790701765.772086:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e46c0. 00000100:00000010:3.0:1790701765.772403:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8ce18e4840. 00000100:00000010:0.0:1790701769.612961:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e2a00. 02000000:00000010:0.0:1790701769.612996:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe280f00. 00000100:00000010:0.0:1790701769.613030:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e0700. 02000000:00000010:0.0:1790701769.613036:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe280000. 00000100:00000010:0.0:1790701769.613044:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd07e1880. 02000000:00000010:0.0:1790701769.613048:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe280c00. 02000000:00000010:0.0:1790701769.613091:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319aa00. 00000400:00000200:0.0:1790701769.614144:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701769.614152:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f370. 00000400:00000010:0.0:1790701769.614163:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701769.614171:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443712, portal 25 00000400:00000010:0.0:1790701769.614179:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbee0. 00000100:00000200:0.0:1790701769.614184:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537443712, offset 0 00000400:00000200:0.0:1790701769.614195:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701769.614204:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701769.614214:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701769.614220:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701769.614226:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701769.614230:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701769.614233:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701769.614238:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [184:184:256:4294967295] 192.168.204.157@tcp LPNI seq info [184:184:8:4294967295] 00000400:00000200:0.0:1790701769.614243:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701769.614255:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701769.614261:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701769.614266:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe280e00. 00000800:00000200:0.0:1790701769.614273:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701769.614281:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701769.614286:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe280e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701769.614303:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:0.0:1790701769.614310:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772cb40. 00000100:00000010:0.0:1790701769.614321:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc772c060. 02000000:00000010:0.0:1790701769.614337:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319ae00. 00000400:00000200:0.0:1790701769.614352:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701769.614354:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f058. 00000400:00000010:0.0:1790701769.614358:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecbaa0. 00000100:00000200:0.0:1790701769.614361:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443968, portal 10 00000400:00000010:0.0:1790701769.614367:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb7f8. 00000100:00000200:0.0:1790701769.614369:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537443968, offset 0 00000400:00000200:0.0:1790701769.614374:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701769.614378:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701769.614381:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701769.614384:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701769.614387:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701769.614389:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701769.614391:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701769.614394:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [185:185:255:4294967295] 192.168.204.157@tcp LPNI seq info [185:185:7:4294967295] 00000400:00000200:0.0:1790701769.614398:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701769.614404:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701769.614408:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701769.614411:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe280300. 00000800:00000200:0.0:1790701769.614415:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701769.614418:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701769.614420:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe280300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701769.614426:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:0.0:1790701769.614449:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc319a400. 00000400:00000200:0.0:1790701769.614453:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:0.0:1790701769.614455:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cc360f0b0. 00000400:00000010:0.0:1790701769.614458:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb220. 00000100:00000200:0.0:1790701769.614461:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537443840, portal 10 00000400:00000010:0.0:1790701769.614465:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cc7ecb880. 00000100:00000200:0.0:1790701769.614467:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537443840, offset 0 00000400:00000200:0.0:1790701769.614471:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:0.0:1790701769.614475:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701769.614484:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:0.0:1790701769.614487:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1790701769.614489:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:0.0:1790701769.614491:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1790701769.614493:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:0.0:1790701769.614495:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [186:186:254:4294967295] 192.168.204.157@tcp LPNI seq info [186:186:6:4294967295] 00000400:00000200:0.0:1790701769.614498:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:0.0:1790701769.614503:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:0.0:1790701769.614507:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:0.0:1790701769.614510:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cfe280200. 00000800:00000200:0.0:1790701769.614513:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:0.0:1790701769.614517:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:0.0:1790701769.614519:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cfe280200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1790701769.614524:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701769.614733:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701769.614738:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe280e00. 00000400:00000200:3.0:1790701769.614743:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701769.614750:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701769.614766:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbee0 00000400:00000010:3.0:1790701769.614769:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbee0. 00000100:00000200:3.0:1790701769.614780:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e2a00 x1877686537443712/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701769.614815:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701769.614830:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe280300. 00000400:00000200:3.0:1790701769.614833:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701769.614837:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701769.614839:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb7f8 00000400:00000010:3.0:1790701769.614841:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb7f8. 00000100:00000200:3.0:1790701769.614845:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e1880 x1877686537443968/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701769.614908:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701769.614911:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cfe280200. 00000400:00000200:3.0:1790701769.614913:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701769.614917:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701769.614919:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb880 00000400:00000010:3.0:1790701769.614921:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb880. 00000100:00000200:3.0:1790701769.614925:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cd07e0700 x1877686537443840/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701769.623401:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623413:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701769.623424:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623429:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623435:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623439:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105a80 00000400:00000200:2.0:1790701769.623444:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623451:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa6d [1] + 224 00000400:00000200:2.0:1790701769.623464:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701769.623469:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701769.623485:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e1880 x1877686537443968/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701769.623564:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623568:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701769.623573:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623576:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623579:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623582:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105a00 00000400:00000200:2.0:1790701769.623585:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623589:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa75 [1] + 224 00000400:00000200:2.0:1790701769.623596:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1790701769.623600:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:2.0:1790701769.623607:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e0700 x1877686537443840/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701769.623621:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623625:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:2.0:1790701769.623630:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623632:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623635:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623637:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105980 00000400:00000200:2.0:1790701769.623641:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:2.0:1790701769.623646:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0xa65 [1] + 224 00000400:00000010:0.0:1790701769.623804:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f0b0. 00000400:00000200:0.0:1790701769.623811:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecb220 00000400:00000010:0.0:1790701769.623814:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecb220. 00000400:00000200:2.0:1790701769.623851:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1790701769.623857:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e0700 x1877686537443840/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1790701769.623863:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:0.0:1790701769.623878:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e0700 x1877686537443840/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1790701769.623879:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cd07e2a00 x1877686537443712/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701769.623936:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f370. 00000400:00000200:0.0:1790701769.623940:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbcc0 00000400:00000010:0.0:1790701769.623942:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbcc0. 00000100:00000200:0.0:1790701769.623947:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e2a00 x1877686537443712/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701769.623957:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e2a00 x1877686537443712/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1790701769.623968:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cc360f058. 00000400:00000200:0.0:1790701769.623979:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cc7ecbaa0 00000400:00000010:0.0:1790701769.623981:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cc7ecbaa0. 00000100:00000200:0.0:1790701769.623988:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cd07e1880 x1877686537443968/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1790701769.623997:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cd07e1880 x1877686537443968/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701785 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1790701769.624031:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319a400. 02000000:00000010:0.0:1790701769.624037:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe280000. 00000100:00000010:0.0:1790701769.624042:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e0700. 02000000:00000010:0.0:1790701769.624046:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319aa00. 02000000:00000010:0.0:1790701769.624049:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe280f00. 00000100:00000010:0.0:1790701769.624051:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e2a00. 02000000:00000010:0.0:1790701769.624055:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc319ae00. 02000000:00000010:0.0:1790701769.624057:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe280c00. 00000100:00000010:0.0:1790701769.624060:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd07e1880. 00000100:00000010:3.0:1790701770.892666:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772c060. 00000100:00000010:2.0:1790701770.894983:0:135704:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc772cb40. 00000001:02000400:2.0:1790701772.122082:0:140048:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 617: lnet_legacy2yaml: round-trip tunables from LNet to modprobe to YAML ========================================================== 13:09:30 (1790701770) 00000001:02000400:3.0:1790701772.419460:0:140062:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000001:02000400:1.0:1790701774.170318:0:140191:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_617: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 00000100:00000010:1.0:1790701774.738134:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701774.738158:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701774.738201:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f4e00. 02000000:00000010:1.0:1790701774.738206:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5d00. 00000100:00000010:1.0:1790701774.738214:0:462:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca3f7480. 02000000:00000010:1.0:1790701774.738218:0:462:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc56d5800. 02000000:00000010:1.0:1790701774.738257:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acda00. 00000400:00000200:1.0:1790701774.738268:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701774.738272:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1856e0. 00000400:00000010:1.0:1790701774.738279:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b26e8. 00000100:00000200:1.0:1790701774.738288:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537444224, portal 10 00000400:00000010:1.0:1790701774.738296:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2cc0. 00000100:00000200:1.0:1790701774.738300:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537444224, offset 0 00000400:00000200:1.0:1790701774.738310:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701774.738319:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701774.738327:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701774.738333:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701774.738346:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701774.738350:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701774.738353:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701774.738357:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [187:187:256:4294967295] 192.168.204.157@tcp LPNI seq info [187:187:8:4294967295] 00000400:00000200:1.0:1790701774.738362:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701774.738372:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701774.738379:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701774.738384:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5b00. 00000800:00000200:1.0:1790701774.738392:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701774.738522:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701774.738528:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701774.738546:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000100:00000010:1.0:1790701774.738553:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cdea0. 00000100:00000010:1.0:1790701774.738567:0:123344:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cfe2cd540. 02000000:00000010:1.0:1790701774.738598:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acc400. 00000400:00000200:1.0:1790701774.738603:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701774.738605:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe185370. 00000400:00000010:1.0:1790701774.738629:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2440. 00000100:00000200:1.0:1790701774.738633:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537444096, portal 25 00000400:00000010:1.0:1790701774.738637:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2a18. 00000100:00000200:1.0:1790701774.738640:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537444096, offset 0 00000400:00000200:1.0:1790701774.738646:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701774.738650:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701774.738671:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701774.738674:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701774.738676:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701774.738679:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701774.738681:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701774.738684:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [188:188:255:4294967295] 192.168.204.157@tcp LPNI seq info [188:188:7:4294967295] 00000400:00000200:1.0:1790701774.738688:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701774.738694:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701774.738706:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701774.738710:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5600. 00000800:00000200:1.0:1790701774.738714:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701774.738719:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701774.738721:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701774.738726:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 02000000:00000010:1.0:1790701774.738741:0:123344:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc7acd200. 00000400:00000200:1.0:1790701774.738745:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000010:1.0:1790701774.738748:0:123344:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cfe1850b0. 00000400:00000010:1.0:1790701774.738752:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2110. 00000100:00000200:1.0:1790701774.738755:0:123344:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537444352, portal 10 00000400:00000010:1.0:1790701774.738760:0:123344:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cec2b2770. 00000100:00000200:1.0:1790701774.738762:0:123344:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537444352, offset 0 00000400:00000200:1.0:1790701774.738766:0:123344:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp 00000400:00000200:1.0:1790701774.738769:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701774.738772:0:123344:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.157@tcp local destination 00000400:00000200:1.0:1790701774.738774:0:123344:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1790701774.738777:0:123344:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp 00000400:00000200:1.0:1790701774.738779:0:123344:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1790701774.738781:0:123344:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp 00000400:00000200:1.0:1790701774.738784:0:123344:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [189:189:254:4294967295] 192.168.204.157@tcp LPNI seq info [189:189:6:4294967295] 00000400:00000200:1.0:1790701774.738788:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:1.0:1790701774.738794:0:123344:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) : PUT try# 0 00000800:00000200:1.0:1790701774.738798:0:123344:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp 00000800:00000010:1.0:1790701774.738801:0:123344:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc56d5c00. 00000800:00000200:1.0:1790701774.738804:0:123344:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5) 00000800:00000200:1.0:1790701774.738808:0:123344:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988 00000800:00000200:1.0:1790701774.738810:0:123344:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc56d5c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1790701774.738814:0:123344:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000800:00000200:3.0:1790701774.739051:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701774.739057:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5b00. 00000400:00000200:3.0:1790701774.739062:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701774.739068:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701774.739074:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2cc0 00000400:00000010:3.0:1790701774.739077:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2cc0. 00000100:00000200:3.0:1790701774.739088:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f4e00 x1877686537444224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701774.739114:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701774.739116:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5600. 00000400:00000200:3.0:1790701774.739118:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701774.739121:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701774.739131:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2a18 00000400:00000010:3.0:1790701774.739132:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2a18. 00000100:00000200:3.0:1790701774.739137:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f5f80 x1877686537444096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1790701774.739213:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1790701774.739216:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc56d5c00. 00000400:00000200:3.0:1790701774.739219:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701774.739223:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000400:00000200:3.0:1790701774.739225:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2770 00000400:00000010:3.0:1790701774.739227:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2770. 00000100:00000200:3.0:1790701774.739232:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0 req@ffff8f8cca3f7480 x1877686537444352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701774.745904:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.745913:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701774.745923:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.745928:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.745933:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.745937:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7105b00 00000400:00000200:3.0:1790701774.745942:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.745948:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0xa85 [1] + 224 00000400:00000200:3.0:1790701774.745963:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701774.745968:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701774.745986:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f5f80 x1877686537444096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1790701774.746548:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.746554:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701774.746559:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.746562:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.746566:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.746569:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105b80 00000400:00000200:3.0:1790701774.746583:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.746588:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa7d [1] + 224 00000400:00000200:3.0:1790701774.746598:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701774.746602:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701774.746609:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f4e00 x1877686537444224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701774.747236:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe185370. 00000400:00000200:1.0:1790701774.747248:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2440 00000400:00000010:1.0:1790701774.747251:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2440. 00000100:00000200:1.0:1790701774.747259:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f5f80 x1877686537444096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701774.747270:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f5f80 x1877686537444096/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701774.747297:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1856e0. 00000400:00000200:1.0:1790701774.747300:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b26e8 00000400:00000010:1.0:1790701774.747302:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b26e8. 00000100:00000200:1.0:1790701774.747307:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f4e00 x1877686537444224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701774.747314:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f4e00 x1877686537444224/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701774.747338:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acc400. 02000000:00000010:1.0:1790701774.747345:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5100. 00000100:00000010:1.0:1790701774.747350:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f5f80. 02000000:00000010:1.0:1790701774.747355:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acda00. 02000000:00000010:1.0:1790701774.747358:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5d00. 00000100:00000010:1.0:1790701774.747360:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f4e00. 00000400:00000200:3.0:1790701774.755902:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.755915:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me 00000400:00000200:3.0:1790701774.755924:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.755929:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.755936:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.755939:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7105c00 00000400:00000200:3.0:1790701774.755944:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1 00000400:00000200:3.0:1790701774.755951:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0xa8d [1] + 224 00000400:00000200:3.0:1790701774.755965:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1790701774.755971:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK 00000100:00000200:3.0:1790701774.755989:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0 req@ffff8f8cca3f7480 x1877686537444352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1790701774.756414:0:123344:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cfe1850b0. 00000400:00000200:1.0:1790701774.756420:0:123344:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cec2b2110 00000400:00000010:1.0:1790701774.756423:0:123344:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cec2b2110. 00000100:00000200:1.0:1790701774.756431:0:123344:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0 req@ffff8f8cca3f7480 x1877686537444352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1790701774.756439:0:123344:0:(events.c:104:reply_in_callback()) @@@ unlink req@ffff8f8cca3f7480 x1877686537444352/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701790 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1790701774.756478:0:123344:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc7acd200. 02000000:00000010:1.0:1790701774.756484:0:123344:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc56d5800. 00000100:00000010:1.0:1790701774.756490:0:123344:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca3f7480. Debug log: 5201 lines, 5201 kept, 0 dropped, 0 bad.