00000100:00000010:0.0F:1788972642.247590:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972642.247627:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 00000100:00000010:0.0:1788972642.247658:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972642.247663:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1db00. 02000000:00000010:0.0:1788972642.247698:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96800. 00000400:00000200:0.0:1788972642.247708:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972642.247712:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163f20. 00000400:00000010:0.0:1788972642.247719:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972642.247727:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106176, portal 25 00000400:00000010:0.0:1788972642.247733:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084bb0. 00000100:00000200:0.0:1788972642.247737:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179106176, offset 0 00000400:00000200:0.0:1788972642.247746:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972642.247755:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972642.247764:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972642.247769:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972642.247773:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972642.247776:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972642.247779:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972642.247783:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [616:616:256:4294967295] 192.168.203.124@tcp LPNI seq info [616:616:8:4294967295] 00000400:00000200:0.0:1788972642.247788:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972642.247797:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972642.247804:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972642.247808:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:0.0:1788972642.247815:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972642.247823:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972642.247827:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972642.247845:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972642.247849:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e849c0. 00000100:00000010:0.0:1788972642.247859:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb44e84480. 02000000:00000010:0.0:1788972642.247877:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51f96400. 00000400:00000200:0.0:1788972642.247881:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972642.247884:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163aa8. 00000400:00000010:0.0:1788972642.247894:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084f68. 00000100:00000200:0.0:1788972642.247897:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106304, portal 10 00000400:00000010:0.0:1788972642.247902:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480845d8. 00000100:00000200:0.0:1788972642.247904:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179106304, offset 0 00000400:00000200:0.0:1788972642.247909:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972642.247913:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972642.247917:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972642.247919:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972642.247922:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972642.247924:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972642.247926:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972642.247929:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [617:617:255:4294967295] 192.168.203.124@tcp LPNI seq info [617:617:7:4294967295] 00000400:00000200:0.0:1788972642.247933:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972642.247938:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972642.247943:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972642.247946:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:0.0:1788972642.247950:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972642.247954:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972642.247957:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972642.247963:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0F:1788972642.248117:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972642.248128:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:3.0:1788972642.248135:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972642.248139:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972642.248144:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084bb0 00000400:00000010:3.0:1788972642.248146:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084bb0. 00000100:00000200:3.0:1788972642.248155:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179106176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972642.248231:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972642.248233:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:3.0:1788972642.248237:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972642.248240:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972642.248243:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480845d8 00000400:00000010:3.0:1788972642.248245:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480845d8. 00000100:00000200:3.0:1788972642.248249:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179106304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972658 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0F:1788972642.250318:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972642.250340:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972642.250348:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972642.250353:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972642.250357:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972642.250361:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13180 00000400:00000200:2.0:1788972642.250366:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972642.250372:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fa5 [1] + 224 00000400:00000200:2.0:1788972642.250387:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972642.250391:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972642.250403:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179106176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972658 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972642.251924:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163f20. 00000400:00000200:0.0:1788972642.251933:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990 00000400:00000010:0.0:1788972642.251936:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972642.251957:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179106176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972658 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972642.251969:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179106176/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972658 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972642.251995:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96800. 02000000:00000010:0.0:1788972642.252001:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:0.0:1788972642.252005:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000400:00000200:3.0:1788972642.252854:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972642.252860:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972642.252867:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972642.252871:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972642.252876:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972642.252879:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13200 00000400:00000200:3.0:1788972642.252884:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972642.252890:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fad [1] + 224 00000400:00000200:3.0:1788972642.252902:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972642.252906:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972642.252915:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179106304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972658 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972642.253685:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163aa8. 00000400:00000200:0.0:1788972642.253692:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084f68 00000400:00000010:0.0:1788972642.253695:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084f68. 00000100:00000200:0.0:1788972642.253703:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179106304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972658 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972642.253713:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179106304/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972658 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972642.253739:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51f96400. 02000000:00000010:0.0:1788972642.253744:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1db00. 00000100:00000010:0.0:1788972642.253747:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000100:00000010:3.0:1788972642.952322:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e84480. 00000100:00000010:2.0:1788972642.952336:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb44e849c0. 00000100:00000010:3.0:1788972647.369568:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788972647.369591:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1900. 00000100:00000010:3.0:1788972647.369646:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c6680. 02000000:00000010:3.0:1788972647.369652:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1b00. 02000000:00000010:2.0:1788972647.369684:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0200. 00000400:00000200:2.0:1788972647.369696:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 02000000:00000010:3.0:1788972647.369697:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058200. 00000400:00000010:2.0:1788972647.369701:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3268. 00000400:00000010:2.0:1788972647.369709:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0aa0. 00000400:00000200:3.0:1788972647.369710:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972647.369717:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78d68. 00000100:00000200:2.0:1788972647.369717:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106432, portal 25 00000400:00000010:3.0:1788972647.369725:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081110. 00000400:00000010:2.0:1788972647.369726:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660. 00000100:00000200:3.0:1788972647.369731:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106560, portal 10 00000100:00000200:2.0:1788972647.369732:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179106432, offset 0 00000400:00000200:2.0:1788972647.369742:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000010:3.0:1788972647.369750:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081cc0. 00000400:00000200:2.0:1788972647.369754:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000200:3.0:1788972647.369756:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179106560, offset 0 00000400:00000200:2.0:1788972647.369763:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972647.369769:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972647.369773:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972647.369776:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972647.369781:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972647.369784:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972647.369788:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [618:618:256:4294967295] 192.168.203.124@tcp LPNI seq info [618:618:8:4294967295] 00000400:00000200:2.0:1788972647.369792:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972647.369800:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972647.369806:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972647.369818:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44c0d700. 00000800:00000200:2.0:1788972647.369824:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972647.369831:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972647.369835:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44c0d700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972647.369849:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972647.369854:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11300. 00000100:00000010:2.0:1788972647.369863:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11ba0. 00000800:00000200:2.0:1788972647.369964:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972647.369968:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44c0d700. 00000400:00000200:2.0:1788972647.369972:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972647.369976:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972647.371280:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.371290:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972647.371294:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972647.371298:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972647.372329:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972647.372333:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972647.372337:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [619:619:255:4294967295] 192.168.203.124@tcp LPNI seq info [619:619:7:4294967295] 00000400:00000200:3.0:1788972647.372344:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372355:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972647.372363:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972647.372371:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1f00. 00000800:00000200:3.0:1788972647.372390:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972647.372397:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972647.372402:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972647.372412:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372462:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372466:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972647.372474:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372477:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372481:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372485:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13280 00000400:00000200:3.0:1788972647.372503:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.372510:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fb5 [1] + 224 00000400:00000200:3.0:1788972647.372524:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972647.372528:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972647.372541:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179106432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972647.372582:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3268. 00000400:00000200:3.0:1788972647.372587:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0aa0 00000400:00000010:3.0:1788972647.372590:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0aa0. 00000100:00000200:3.0:1788972647.372596:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179106432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972663 ref 2 fl Rpc:RN/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972647.372605:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179106432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972663 ref 2 fl Rpc:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972647.376764:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660 00000400:00000010:2.0:1788972647.376771:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788972647.376787:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179106432/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972663 ref 2 fl UnregRPC:RNU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972647.376915:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972647.376919:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1f00. 00000400:00000200:2.0:1788972647.376925:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972647.376930:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972647.376933:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081cc0 00000400:00000010:2.0:1788972647.376935:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081cc0. 00000100:00000200:2.0:1788972647.376941:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c6680 x1875872179106560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972663 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972647.377065:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0200. 02000000:00000010:3.0:1788972647.377073:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1900. 00000100:00000010:3.0:1788972647.377077:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000400:00000200:3.0:1788972647.379872:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.379881:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972647.379888:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.379893:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.379897:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.379902:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13300 00000400:00000200:3.0:1788972647.379908:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972647.379914:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fbd [1] + 224 00000400:00000200:3.0:1788972647.379927:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972647.379931:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972647.379942:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c6680 x1875872179106560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972663 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972647.380052:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78d68. 00000400:00000200:2.0:1788972647.380058:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081110 00000400:00000010:2.0:1788972647.380060:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081110. 00000100:00000200:2.0:1788972647.380066:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c6680 x1875872179106560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972663 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972647.380075:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c6680 x1875872179106560/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972663 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972647.380102:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058200. 02000000:00000010:2.0:1788972647.380124:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1b00. 00000100:00000010:2.0:1788972647.380130:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c6680. 00000100:00000010:2.0:1788972648.071644:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11300. 00000100:00000010:3.0:1788972648.072940:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11ba0. 00000001:02000400:1.0F:1788972651.309223:0:126761:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 550: DD-off: keep route up when gw reachable via another local NI ========================================================== 12:50:50 (1788972650) 00000100:00000010:0.0:1788972652.490550:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972652.490572:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 02000000:00000010:1.0:1788972652.490643:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69a00. 00000400:00000200:1.0:1788972652.490654:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972652.490658:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1d10. 00000400:00000010:1.0:1788972652.490665:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588880. 00000100:00000200:1.0:1788972652.490672:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106688, portal 25 00000400:00000010:1.0:1788972652.490678:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588dd0. 00000100:00000200:1.0:1788972652.490681:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179106688, offset 0 00000400:00000200:1.0:1788972652.490690:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972652.490699:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972652.490707:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972652.490723:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972652.490728:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972652.490731:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972652.490735:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972652.490738:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [620:620:256:4294967295] 192.168.203.124@tcp LPNI seq info [620:620:8:4294967295] 00000400:00000200:1.0:1788972652.490742:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972652.490753:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972652.490759:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972652.490765:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49714800. 00000800:00000200:1.0:1788972652.490781:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972652.490789:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972652.490793:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49714800 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:0.0:1788972652.490975:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972652.490993:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200. 02000000:00000010:0.0:1788972652.491024:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2e00. 00000400:00000200:0.0:1788972652.491029:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972652.491033:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163aa8. 00000400:00000010:0.0:1788972652.491037:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084f68. 00000100:00000200:0.0:1788972652.491040:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106816, portal 10 00000400:00000010:0.0:1788972652.491046:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972652.491049:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179106816, offset 0 00000400:00000200:0.0:1788972652.491054:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972652.491058:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972652.491062:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972652.491065:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972652.491067:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972652.491070:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972652.491072:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972652.491074:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [621:621:255:4294967295] 192.168.203.124@tcp LPNI seq info [621:621:7:4294967295] 00000400:00000200:0.0:1788972652.491078:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972652.491084:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972652.491098:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972652.491101:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:0.0:1788972652.491105:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972652.491110:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972652.491112:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972652.491134:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972652.491139:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb519f75a0. 00000100:00000010:1.0:1788972652.491152:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51ab84e0. 00000400:00000200:0.0:1788972652.491446:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972652.492628:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972652.492635:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49714800. 00000400:00000200:2.0:1788972652.492642:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972652.492649:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972652.492655:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588dd0 00000400:00000010:2.0:1788972652.492657:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588dd0. 00000100:00000200:2.0:1788972652.492667:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179106688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972652.492761:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972652.492764:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:2.0:1788972652.493087:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972652.493092:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972652.493096:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990 00000400:00000010:2.0:1788972652.493098:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990. 00000100:00000200:2.0:1788972652.493105:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179106816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972668 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972652.498708:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498717:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972652.498727:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498731:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498736:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498739:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13380 00000400:00000200:3.0:1788972652.498744:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498750:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fd5 [1] + 224 00000400:00000200:3.0:1788972652.498761:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972652.498765:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972652.498789:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179106688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972668 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972652.498822:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498826:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972652.498830:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498832:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498834:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972652.498837:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13400 00000400:00000010:0.0:1788972652.498840:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1d10. 00000400:00000200:3.0:1788972652.498841:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972652.498845:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588880 00000400:00000010:0.0:1788972652.498847:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588880. 00000100:00000200:0.0:1788972652.498854:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179106688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972668 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972652.498864:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179106688/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972668 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972652.498890:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69a00. 02000000:00000010:0.0:1788972652.498896:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:0.0:1788972652.498902:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000400:00000200:3.0:1788972652.499024:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fdd [1] + 224 00000400:00000200:3.0:1788972652.499036:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972652.499039:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972652.499046:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179106816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972668 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972652.499193:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163aa8. 00000400:00000200:1.0:1788972652.499199:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084f68 00000400:00000010:1.0:1788972652.499200:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084f68. 00000100:00000200:1.0:1788972652.499207:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179106816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972668 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972652.499226:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179106816/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972668 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972652.499245:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2e00. 02000000:00000010:1.0:1788972652.499250:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200. 00000100:00000010:1.0:1788972652.499253:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000001:02000400:1.0:1788972652.786824:0:126897:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_550 Need at least 2 local interfaces for multi-rail 00000100:00000010:2.0:1788972653.191362:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb519f75a0. 00000100:00000010:3.0:1788972653.192819:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51ab84e0. 00000001:02000400:2.0:1788972654.483850:0:127050:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 600: lnet_legacy2yaml: basic ip2nets conversion ========================================================== 12:50:53 (1788972653) 00000100:00000010:1.0:1788972657.610694:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4a80. 02000000:00000010:1.0:1788972657.610726:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49714900. 00000100:00000010:1.0:1788972657.610772:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb5500. 02000000:00000010:1.0:1788972657.610777:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb49714800. 02000000:00000010:1.0:1788972657.610818:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51151000. 00000400:00000200:1.0:1788972657.610829:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972657.610835:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972657.610843:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788972657.610852:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107072, portal 10 00000400:00000010:1.0:1788972657.610861:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588110. 00000100:00000200:1.0:1788972657.610866:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179107072, offset 0 00000400:00000200:1.0:1788972657.610877:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972657.610888:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972657.610897:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972657.610904:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972657.610909:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972657.610913:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972657.610917:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972657.610921:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [622:622:256:4294967295] 192.168.203.124@tcp LPNI seq info [622:622:8:4294967295] 00000400:00000200:1.0:1788972657.611011:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972657.611024:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972657.611033:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972657.611039:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49714300. 00000800:00000200:1.0:1788972657.611054:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972657.611064:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972657.611069:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49714300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972657.611094:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972657.611099:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43dc0180. 00000100:00000010:1.0:1788972657.611286:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb43dc0720. 02000000:00000010:1.0:1788972657.611349:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51151200. 00000400:00000200:1.0:1788972657.611355:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972657.611358:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1c08. 00000400:00000010:1.0:1788972657.611362:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972657.611366:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179106944, portal 25 00000400:00000010:1.0:1788972657.611370:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445886e8. 00000100:00000200:1.0:1788972657.611372:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179106944, offset 0 00000400:00000200:1.0:1788972657.611376:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972657.611381:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972657.611385:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972657.611388:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972657.611390:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972657.611393:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972657.611395:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972657.611398:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [623:623:255:4294967295] 192.168.203.124@tcp LPNI seq info [623:623:7:4294967295] 00000400:00000200:1.0:1788972657.611401:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972657.611407:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972657.611411:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972657.611414:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb49714e00. 00000800:00000200:1.0:1788972657.611418:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972657.611422:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972657.611425:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb49714e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972657.611429:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972657.611656:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972657.611662:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49714300. 00000400:00000200:3.0:1788972657.611670:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972657.611676:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972657.611682:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588110 00000400:00000010:3.0:1788972657.611690:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588110. 00000100:00000200:3.0:1788972657.611700:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb5500 x1875872179107072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972657.612071:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972657.612075:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb49714e00. 00000400:00000200:3.0:1788972657.612078:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972657.612083:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972657.612086:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445886e8 00000400:00000010:3.0:1788972657.612088:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445886e8. 00000100:00000200:3.0:1788972657.612094:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4a80 x1875872179106944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972673 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972657.616238:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616251:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972657.616262:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616269:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616276:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616281:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13500 00000400:00000200:2.0:1788972657.616287:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616295:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fe5 [1] + 224 00000400:00000200:2.0:1788972657.616518:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972657.616531:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972657.616552:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb5500 x1875872179107072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972673 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972657.616669:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972657.616675:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330 00000400:00000200:2.0:1788972657.616681:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972657.616684:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330. 00000400:00000200:2.0:1788972657.616685:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972657.616690:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000200:1.0:1788972657.616692:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb5500 x1875872179107072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972673 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972657.616693:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616696:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616699:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13480 00000100:00000200:1.0:1788972657.616702:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb5500 x1875872179107072/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972673 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972657.616703:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972657.616708:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1fed [1] + 224 00000400:00000200:2.0:1788972657.616717:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972657.616720:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972657.616726:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4a80 x1875872179106944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972673 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972657.616744:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51151000. 02000000:00000010:1.0:1788972657.616751:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49714800. 00000100:00000010:1.0:1788972657.616758:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb5500. 00000400:00000010:1.0:1788972657.616767:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1c08. 00000400:00000200:1.0:1788972657.616769:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445883b8 00000400:00000010:1.0:1788972657.616775:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972657.616780:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4a80 x1875872179106944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972673 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972657.616788:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4a80 x1875872179106944/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972673 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972657.616799:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51151200. 02000000:00000010:1.0:1788972657.616803:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb49714900. 00000100:00000010:1.0:1788972657.616805:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4a80. 00000100:00000010:3.0:1788972658.313048:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43dc0720. 00000100:00000010:2.0:1788972658.313425:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb43dc0180. 00000001:02000400:3.0:1788972661.640258:0:127622:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 601: lnet_legacy2yaml: basic routes conversion ========================================================== 12:51:00 (1788972660) 00000100:00000010:2.0:1788972662.729490:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413b80. 02000000:00000010:2.0:1788972662.729511:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285800. 00000100:00000010:2.0:1788972662.729546:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410700. 02000000:00000010:2.0:1788972662.729552:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285900. 02000000:00000010:2.0:1788972662.729586:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:2.0:1788972662.729594:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972662.729598:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3e70. 00000400:00000010:2.0:1788972662.729605:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0660. 00000100:00000200:2.0:1788972662.729611:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107200, portal 25 00000400:00000010:2.0:1788972662.729616:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330. 00000100:00000200:2.0:1788972662.729618:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179107200, offset 0 00000400:00000200:2.0:1788972662.729638:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972662.729648:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972662.729656:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972662.729675:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972662.729679:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972662.729683:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972662.729686:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972662.729690:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [624:624:256:4294967295] 192.168.203.124@tcp LPNI seq info [624:624:8:4294967295] 00000400:00000200:2.0:1788972662.729695:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972662.729705:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972662.729711:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972662.729716:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285c00. 00000800:00000200:2.0:1788972662.729723:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972662.729731:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972662.729735:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972662.729748:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972662.729753:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11f00. 00000100:00000010:2.0:1788972662.729762:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11c00. 02000000:00000010:2.0:1788972662.729785:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:2.0:1788972662.729789:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972662.729792:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b37e8. 00000400:00000010:2.0:1788972662.729796:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0880. 00000100:00000200:2.0:1788972662.729799:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107328, portal 10 00000400:00000010:2.0:1788972662.729804:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972662.729807:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179107328, offset 0 00000400:00000200:2.0:1788972662.729812:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972662.729830:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972662.729835:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972662.729837:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972662.729840:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972662.729842:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972662.729844:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972662.729847:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [625:625:255:4294967295] 192.168.203.124@tcp LPNI seq info [625:625:7:4294967295] 00000400:00000200:2.0:1788972662.729850:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972662.729863:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972662.729868:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972662.729872:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285e00. 00000800:00000200:2.0:1788972662.729876:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972662.729881:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972662.729884:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972662.729890:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972662.729950:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972662.729954:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285c00. 00000400:00000200:2.0:1788972662.729961:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972662.729966:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972662.729971:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330 00000400:00000010:2.0:1788972662.729974:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330. 00000100:00000200:2.0:1788972662.729984:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413b80 x1875872179107200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972662.730244:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972662.730247:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285e00. 00000400:00000200:2.0:1788972662.730250:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972662.730254:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972662.730257:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:2.0:1788972662.730259:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972662.730263:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410700 x1875872179107328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972678 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972662.743829:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.743839:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972662.743847:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.743852:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.743857:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.743861:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13580 00000400:00000200:3.0:1788972662.743865:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.743871:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x1ff5 [1] + 224 00000400:00000200:3.0:1788972662.743883:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972662.743888:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972662.743900:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413b80 x1875872179107200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972678 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972662.744418:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.744423:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972662.744428:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.744431:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.744449:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.744452:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13600 00000400:00000200:3.0:1788972662.744456:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972662.744461:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2005 [1] + 224 00000400:00000200:3.0:1788972662.744472:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972662.744476:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972662.744484:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410700 x1875872179107328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972678 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972662.744516:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b37e8. 00000400:00000200:3.0:1788972662.744521:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0880 00000400:00000010:3.0:1788972662.744523:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0880. 00000100:00000200:3.0:1788972662.744529:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410700 x1875872179107328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972678 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972662.744539:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410700 x1875872179107328/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972678 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972662.744568:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3e70. 00000400:00000200:3.0:1788972662.744571:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0660 00000400:00000010:3.0:1788972662.744573:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0660. 00000100:00000200:3.0:1788972662.744577:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413b80 x1875872179107200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972678 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972662.744585:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413b80 x1875872179107200/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972678 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972662.744599:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:3.0:1788972662.744605:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285900. 00000100:00000010:3.0:1788972662.744609:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410700. 02000000:00000010:3.0:1788972662.744613:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:3.0:1788972662.744616:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285800. 00000100:00000010:3.0:1788972662.744618:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413b80. 00000100:00000010:3.0:1788972663.431362:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11c00. 00000100:00000010:2.0:1788972663.431516:0:115691:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11f00. 00000100:00000010:3.0:1788972667.848496:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4e00. 02000000:00000010:3.0:1788972667.848516:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1300. 00000100:00000010:3.0:1788972667.848541:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c4000. 02000000:00000010:3.0:1788972667.848545:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1d00. 02000000:00000010:3.0:1788972667.848657:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f4e00. 00000400:00000200:3.0:1788972667.848666:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972667.848670:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789f8. 00000400:00000010:3.0:1788972667.848677:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480813b8. 00000100:00000200:3.0:1788972667.848684:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107456, portal 25 00000400:00000010:3.0:1788972667.848689:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972667.848692:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179107456, offset 0 00000400:00000200:3.0:1788972667.848701:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972667.848710:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.848718:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972667.848724:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972667.848728:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972667.848732:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972667.848735:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972667.848744:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [626:626:256:4294967295] 192.168.203.124@tcp LPNI seq info [626:626:8:4294967295] 00000400:00000200:3.0:1788972667.848748:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.848759:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972667.848765:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972667.848770:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1700. 00000800:00000200:3.0:1788972667.848777:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972667.848786:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972667.848790:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972667.848808:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972667.848813:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9ae0. 00000100:00000010:3.0:1788972667.848824:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9c60. 02000000:00000010:3.0:1788972667.848847:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4a0f5000. 00000400:00000200:3.0:1788972667.848864:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972667.848882:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78580. 00000400:00000010:3.0:1788972667.848887:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081330. 00000100:00000200:3.0:1788972667.848890:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107584, portal 10 00000400:00000010:3.0:1788972667.848896:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972667.848899:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179107584, offset 0 00000400:00000200:3.0:1788972667.848904:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972667.848909:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.848913:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972667.848917:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972667.848920:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972667.848923:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972667.848925:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972667.848928:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [627:627:255:4294967295] 192.168.203.124@tcp LPNI seq info [627:627:7:4294967295] 00000400:00000200:3.0:1788972667.848932:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.848938:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972667.848943:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972667.848947:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1600. 00000800:00000200:3.0:1788972667.848951:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972667.848956:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972667.848959:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972667.848965:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972667.849664:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972667.849669:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1700. 00000400:00000200:3.0:1788972667.849675:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972667.849681:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972667.849687:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440 00000400:00000010:3.0:1788972667.849689:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972667.849698:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4e00 x1875872179107456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972667.849775:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972667.849778:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1600. 00000400:00000200:3.0:1788972667.849787:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972667.849791:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972667.849794:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972667.849796:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972667.849801:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c4000 x1875872179107584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972683 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972667.854994:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855004:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972667.855014:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855020:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855025:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855029:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13700 00000400:00000200:3.0:1788972667.855034:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855041:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2025 [1] + 224 00000400:00000200:3.0:1788972667.855051:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972667.855056:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972667.855072:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4000 x1875872179107584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972683 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972667.855101:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855106:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972667.855111:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855114:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855118:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855121:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13680 00000400:00000200:3.0:1788972667.855126:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972667.855132:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2015 [1] + 224 00000400:00000200:3.0:1788972667.855141:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972667.855146:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972667.855153:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c4e00 x1875872179107456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972683 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972667.855361:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78580. 00000400:00000200:2.0:1788972667.855368:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081330 00000400:00000010:2.0:1788972667.855370:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081330. 00000100:00000200:2.0:1788972667.855378:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4000 x1875872179107584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972683 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972667.855390:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4000 x1875872179107584/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972683 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972667.855425:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789f8. 00000400:00000200:2.0:1788972667.855427:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480813b8 00000400:00000010:2.0:1788972667.855429:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480813b8. 00000100:00000200:2.0:1788972667.855432:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c4e00 x1875872179107456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972683 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972667.855437:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c4e00 x1875872179107456/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972683 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972667.855448:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f5000. 02000000:00000010:2.0:1788972667.855452:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1d00. 00000100:00000010:2.0:1788972667.855897:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4000. 02000000:00000010:2.0:1788972667.855904:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4a0f4e00. 02000000:00000010:2.0:1788972667.855908:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1300. 00000100:00000010:2.0:1788972667.855911:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c4e00. 00000100:00000010:2.0:1788972668.551531:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9ae0. 00000100:00000010:3.0:1788972668.552359:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9c60. 00000001:02000400:3.0:1788972668.575424:0:128195:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 602: lnet_legacy2yaml: routes with priority ========================================================== 12:51:07 (1788972667) 00000100:00000010:0.0:1788972672.974125:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972672.974145:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1dd00. 00000100:00000010:0.0:1788972672.974176:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972672.974181:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1df00. 02000000:00000010:0.0:1788972672.974211:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017c00. 00000400:00000200:0.0:1788972672.974220:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972672.974225:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163f20. 00000400:00000010:0.0:1788972672.974232:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972672.974239:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107712, portal 25 00000400:00000010:0.0:1788972672.974245:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788972672.974248:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179107712, offset 0 00000400:00000200:0.0:1788972672.974256:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972672.974264:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972672.974271:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972672.974284:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972672.974288:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972672.974291:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972672.974294:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972672.974405:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [628:628:256:4294967295] 192.168.203.124@tcp LPNI seq info [628:628:8:4294967295] 00000400:00000200:0.0:1788972672.974413:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972672.974423:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972672.974430:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972672.974434:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d200. 00000800:00000200:0.0:1788972672.974441:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972672.974449:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972672.974453:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972672.974501:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972672.974507:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb413241e0. 00000100:00000010:0.0:1788972672.974516:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324000. 02000000:00000010:0.0:1788972672.974535:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016400. 00000400:00000200:0.0:1788972672.974539:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972672.974541:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163a50. 00000400:00000010:0.0:1788972672.974545:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972672.974547:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107840, portal 10 00000400:00000010:0.0:1788972672.974552:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084330. 00000100:00000200:0.0:1788972672.974554:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179107840, offset 0 00000400:00000200:0.0:1788972672.974558:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972672.974562:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972672.974565:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972672.974568:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972672.974571:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972672.974573:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972672.974575:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972672.974578:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [629:629:255:4294967295] 192.168.203.124@tcp LPNI seq info [629:629:7:4294967295] 00000400:00000200:0.0:1788972672.974581:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972672.974593:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972672.974598:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972672.974602:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1de00. 00000800:00000200:0.0:1788972672.974606:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972672.974610:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972672.974613:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1de00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972672.974619:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972672.979490:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972672.979500:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d200. 00000400:00000200:3.0:1788972672.979506:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972672.979516:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972672.979523:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:3.0:1788972672.979525:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:3.0:1788972672.979537:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179107712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972688 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972672.979606:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972672.979609:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1de00. 00000400:00000200:3.0:1788972672.979612:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972672.979615:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972672.979618:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084330 00000400:00000010:3.0:1788972672.979619:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084330. 00000100:00000200:3.0:1788972672.979624:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179107840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972688 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972672.984147:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984156:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972672.984164:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984168:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984173:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984178:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13780 00000400:00000200:3.0:1788972672.984183:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984294:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2035 [1] + 224 00000400:00000200:3.0:1788972672.984456:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972672.984462:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972672.984476:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179107712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972688 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972672.984520:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984524:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972672.984528:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984531:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984534:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984536:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13800 00000400:00000200:3.0:1788972672.984540:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972672.984544:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x203d [1] + 224 00000400:00000200:3.0:1788972672.984551:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972672.984555:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972672.984561:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179107840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972688 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972672.985577:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163a50. 00000400:00000200:0.0:1788972672.985584:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:0.0:1788972672.985586:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972672.985597:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179107840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972688 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972672.985606:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179107840/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972688 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972672.985632:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163f20. 00000400:00000200:0.0:1788972672.985635:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:0.0:1788972672.985637:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972672.985641:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179107712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972688 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972672.985649:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179107712/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972688 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972672.985674:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016400. 02000000:00000010:0.0:1788972672.985679:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1df00. 00000100:00000010:0.0:1788972672.985684:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972672.985688:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017c00. 02000000:00000010:0.0:1788972672.985690:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1dd00. 00000100:00000010:0.0:1788972672.985692:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000100:00000010:2.0:1788972673.671373:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb413241e0. 00000100:00000010:3.0:1788972673.671478:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324000. 00000001:02000400:3.0:1788972675.604473:0:128767:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 603: lnet_legacy2yaml: default hop flag ========================================================== 12:51:14 (1788972674) 00000100:00000010:3.0:1788972678.092848:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788972678.092870:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1500. 00000100:00000010:3.0:1788972678.092912:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972678.092917:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1d00. 02000000:00000010:3.0:1788972678.092949:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058400. 00000400:00000200:3.0:1788972678.092958:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972678.092963:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789f8. 00000400:00000010:3.0:1788972678.092970:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972678.092977:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108096, portal 10 00000400:00000010:3.0:1788972678.092985:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972678.092988:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179108096, offset 0 00000400:00000200:3.0:1788972678.092996:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972678.093004:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.093012:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972678.093024:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972678.093029:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972678.093032:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972678.093035:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972678.093039:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [630:630:256:4294967295] 192.168.203.124@tcp LPNI seq info [630:630:8:4294967295] 00000400:00000200:3.0:1788972678.093044:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.093053:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972678.093059:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972678.093064:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1f00. 00000800:00000200:3.0:1788972678.093070:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972678.093077:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972678.093081:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972678.093096:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972678.093100:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd98a0. 00000100:00000010:3.0:1788972678.093109:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9ba0. 02000000:00000010:3.0:1788972678.093127:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54059800. 00000400:00000200:3.0:1788972678.093131:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972678.093134:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78580. 00000400:00000010:3.0:1788972678.093137:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972678.093140:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179107968, portal 25 00000400:00000010:3.0:1788972678.093143:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972678.093145:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179107968, offset 0 00000400:00000200:3.0:1788972678.093150:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972678.093154:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.093157:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972678.093160:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972678.093162:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972678.093164:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972678.093166:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972678.093169:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [631:631:255:4294967295] 192.168.203.124@tcp LPNI seq info [631:631:7:4294967295] 00000400:00000200:3.0:1788972678.093173:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.093184:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972678.093189:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972678.093192:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100. 00000800:00000200:3.0:1788972678.093196:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972678.093201:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972678.093204:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972678.093208:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972678.093252:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972678.093256:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1f00. 00000400:00000200:3.0:1788972678.093261:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972678.093265:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972678.093270:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440 00000400:00000010:3.0:1788972678.093273:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972678.093281:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5880 x1875872179108096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972678.094548:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972678.094553:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100. 00000400:00000200:3.0:1788972678.094557:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972678.094563:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972678.094568:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972678.094570:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972678.094579:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179107968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972694 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972678.100781:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.100790:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972678.100810:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.100815:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.100820:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.100824:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13880 00000400:00000200:3.0:1788972678.100829:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.100835:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2055 [1] + 224 00000400:00000200:3.0:1788972678.100856:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972678.100860:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972678.100874:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179107968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972694 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972678.100917:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78580. 00000400:00000200:3.0:1788972678.100922:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972678.100924:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972678.100930:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179107968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972694 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972678.100939:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179107968/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972694 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972678.100965:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54059800. 02000000:00000010:3.0:1788972678.100972:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1500. 00000100:00000010:3.0:1788972678.100977:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 00000400:00000200:3.0:1788972678.103752:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.103760:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972678.103768:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.103772:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.103778:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.103782:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13900 00000400:00000200:3.0:1788972678.103787:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972678.103792:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2045 [1] + 224 00000400:00000200:3.0:1788972678.103818:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972678.103823:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972678.103836:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5880 x1875872179108096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972694 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972678.103878:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789f8. 00000400:00000200:3.0:1788972678.103883:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972678.103885:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972678.103901:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5880 x1875872179108096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972694 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972678.103909:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5880 x1875872179108096/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972694 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972678.103940:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058400. 02000000:00000010:3.0:1788972678.103946:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1d00. 00000100:00000010:3.0:1788972678.103951:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880. 00000100:00000010:3.0:1788972678.791423:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9ba0. 00000100:00000010:2.0:1788972678.791527:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd98a0. 00000001:02000400:0.0:1788972681.910282:0:129339:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 604: lnet_legacy2yaml: default hop with explicit value ========================================================== 12:51:20 (1788972680) 00000100:00000010:0.0:1788972683.211244:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353fb80. 02000000:00000010:0.0:1788972683.211282:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d200. 00000100:00000010:0.0:1788972683.211437:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ce00. 02000000:00000010:0.0:1788972683.211443:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1de00. 02000000:00000010:0.0:1788972683.211998:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a600. 00000400:00000200:0.0:1788972683.212009:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972683.212013:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163f20. 00000400:00000010:0.0:1788972683.212019:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084220. 00000100:00000200:0.0:1788972683.212026:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108224, portal 25 00000400:00000010:0.0:1788972683.212031:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084aa0. 00000100:00000200:0.0:1788972683.212034:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179108224, offset 0 00000400:00000200:0.0:1788972683.212041:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972683.212049:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972683.212056:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972683.212061:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972683.212065:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972683.212069:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972683.212071:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972683.212074:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [632:632:256:4294967295] 192.168.203.124@tcp LPNI seq info [632:632:8:4294967295] 00000400:00000200:0.0:1788972683.212078:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972683.212087:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972683.212093:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972683.212097:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1df00. 00000800:00000200:0.0:1788972683.212105:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972683.212112:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972683.212116:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1df00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972683.212138:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972683.212143:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb413244e0. 00000100:00000010:0.0:1788972683.212153:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324720. 02000000:00000010:0.0:1788972683.212174:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a000. 00000400:00000200:0.0:1788972683.212178:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972683.212180:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163a50. 00000400:00000010:0.0:1788972683.212192:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084770. 00000100:00000200:0.0:1788972683.212195:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108352, portal 10 00000400:00000010:0.0:1788972683.212201:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084330. 00000100:00000200:0.0:1788972683.212203:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179108352, offset 0 00000400:00000200:0.0:1788972683.212208:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972683.212212:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972683.212215:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972683.212217:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972683.212219:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972683.212221:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972683.212223:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972683.212226:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [633:633:255:4294967295] 192.168.203.124@tcp LPNI seq info [633:633:7:4294967295] 00000400:00000200:0.0:1788972683.212229:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972683.212234:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972683.212238:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972683.212241:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d900. 00000800:00000200:0.0:1788972683.212245:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972683.212249:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972683.212251:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972683.212257:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972683.212618:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972683.212624:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1df00. 00000400:00000200:3.0:1788972683.212628:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972683.212633:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972683.212638:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084aa0 00000400:00000010:3.0:1788972683.212640:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084aa0. 00000100:00000200:3.0:1788972683.212649:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353fb80 x1875872179108224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972699 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972683.212727:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972683.212731:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d900. 00000400:00000200:3.0:1788972683.212734:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972683.212737:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972683.212740:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084330 00000400:00000010:3.0:1788972683.212742:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084330. 00000100:00000200:3.0:1788972683.212747:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ce00 x1875872179108352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972699 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972683.217514:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.217524:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972683.217532:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.217536:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.217541:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.217545:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13a00 00000400:00000200:3.0:1788972683.217550:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.217565:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x206d [1] + 224 00000400:00000200:3.0:1788972683.217580:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972683.217584:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972683.217598:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ce00 x1875872179108352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972699 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972683.217785:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163a50. 00000400:00000200:1.0:1788972683.217791:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084770 00000400:00000010:1.0:1788972683.217793:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084770. 00000100:00000200:1.0:1788972683.217800:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ce00 x1875872179108352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972699 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972683.217810:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ce00 x1875872179108352/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972699 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972683.217840:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a000. 02000000:00000010:1.0:1788972683.217846:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1de00. 00000100:00000010:1.0:1788972683.217850:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ce00. 00000400:00000200:3.0:1788972683.218530:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.218535:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972683.218540:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.218543:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.218547:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.218550:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13980 00000400:00000200:3.0:1788972683.218555:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972683.218560:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2065 [1] + 224 00000400:00000200:3.0:1788972683.218569:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972683.218573:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972683.218582:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353fb80 x1875872179108224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972699 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972683.218894:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163f20. 00000400:00000200:1.0:1788972683.218909:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084220 00000400:00000010:1.0:1788972683.218911:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084220. 00000100:00000200:1.0:1788972683.218917:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353fb80 x1875872179108224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972699 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972683.218927:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353fb80 x1875872179108224/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972699 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972683.218944:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a600. 02000000:00000010:1.0:1788972683.218949:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d200. 00000100:00000010:1.0:1788972683.218952:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353fb80. 00000100:00000010:2.0:1788972683.914429:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb413244e0. 00000100:00000010:3.0:1788972683.914558:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324720. 00000100:00000010:0.0:1788972688.328032:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972688.328057:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d700. 00000100:00000010:0.0:1788972688.328092:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972688.328097:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51b1d500. 02000000:00000010:0.0:1788972688.328129:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb54058400. 00000400:00000200:0.0:1788972688.328139:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972688.328144:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163738. 00000400:00000010:0.0:1788972688.328150:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788972688.328157:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108608, portal 10 00000400:00000010:0.0:1788972688.328162:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084f68. 00000100:00000200:0.0:1788972688.328166:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179108608, offset 0 00000400:00000200:0.0:1788972688.328186:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972688.328194:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972688.328202:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972688.328206:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972688.328211:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972688.328215:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972688.328217:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972688.328221:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [634:634:256:4294967295] 192.168.203.124@tcp LPNI seq info [634:634:8:4294967295] 00000400:00000200:0.0:1788972688.328225:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972688.328235:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972688.328241:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972688.328246:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1db00. 00000800:00000200:0.0:1788972688.328252:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972688.328258:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972688.328263:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1db00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972688.328284:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972688.328289:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324c60. 00000100:00000010:0.0:1788972688.328316:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324ba0. 02000000:00000010:0.0:1788972688.328339:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb48185800. 00000400:00000200:0.0:1788972688.328343:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972688.328346:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e18. 00000400:00000010:0.0:1788972688.328349:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972688.328352:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108480, portal 25 00000400:00000010:0.0:1788972688.328355:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480845d8. 00000100:00000200:0.0:1788972688.328357:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179108480, offset 0 00000400:00000200:0.0:1788972688.328361:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972688.328364:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972688.328367:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972688.328370:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972688.328372:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972688.328373:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972688.328382:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972688.328384:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [635:635:255:4294967295] 192.168.203.124@tcp LPNI seq info [635:635:7:4294967295] 00000400:00000200:0.0:1788972688.328388:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972688.328395:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972688.328400:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972688.328403:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51b1d300. 00000800:00000200:0.0:1788972688.328407:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972688.328411:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972688.328414:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51b1d300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972688.328418:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972688.329380:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972688.329387:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1db00. 00000400:00000200:3.0:1788972688.329391:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972688.329398:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972688.329403:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084f68 00000400:00000010:3.0:1788972688.329405:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084f68. 00000100:00000200:3.0:1788972688.329414:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179108608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972688.329788:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972688.329791:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51b1d300. 00000400:00000200:3.0:1788972688.329794:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972688.329798:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972688.329811:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480845d8 00000400:00000010:3.0:1788972688.329813:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480845d8. 00000100:00000200:3.0:1788972688.329819:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179108480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972704 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972688.338835:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338844:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972688.338854:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338858:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338864:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338868:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13b00 00000400:00000200:3.0:1788972688.338873:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338879:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2075 [1] + 224 00000400:00000200:3.0:1788972688.338890:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972688.338895:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972688.338908:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179108608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972704 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972688.338933:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338937:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972688.338940:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338942:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338945:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338947:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13a80 00000400:00000200:3.0:1788972688.338950:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972688.338954:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x207d [1] + 224 00000400:00000200:3.0:1788972688.338963:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972688.338966:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972688.338971:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179108480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972704 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972688.340372:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e18. 00000400:00000200:0.0:1788972688.340380:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990 00000400:00000010:0.0:1788972688.340382:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972688.340393:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179108480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972704 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972688.340403:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179108480/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972704 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972688.340427:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163738. 00000400:00000200:0.0:1788972688.340430:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:0.0:1788972688.340431:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788972688.340436:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179108608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972704 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972688.340443:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179108608/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972704 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972688.340463:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb48185800. 02000000:00000010:0.0:1788972688.340469:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d700. 00000100:00000010:0.0:1788972688.340485:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972688.340489:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb54058400. 02000000:00000010:0.0:1788972688.340492:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51b1d500. 00000100:00000010:0.0:1788972688.340503:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000100:00000010:3.0:1788972689.031397:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324ba0. 00000100:00000010:2.0:1788972689.031719:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324c60. 00000001:02000400:1.0:1788972689.379874:0:129912:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 605: lnet_legacy2yaml: quote leading star in IP pattern ========================================================== 12:51:27 (1788972687) 00000100:00000010:3.0:1788972693.450817:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972693.450836:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1500. 00000100:00000010:3.0:1788972693.450864:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788972693.450870:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1700. 02000000:00000010:3.0:1788972693.450908:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548600. 00000400:00000200:3.0:1788972693.450918:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972693.450922:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789f8. 00000400:00000010:3.0:1788972693.450929:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972693.450936:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108864, portal 10 00000400:00000010:3.0:1788972693.450944:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972693.450947:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179108864, offset 0 00000400:00000200:3.0:1788972693.450957:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972693.450965:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.450972:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972693.450978:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972693.450982:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972693.450985:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972693.450988:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972693.450991:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [636:636:256:4294967295] 192.168.203.124@tcp LPNI seq info [636:636:8:4294967295] 00000400:00000200:3.0:1788972693.450995:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.451003:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972693.451009:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972693.451014:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1600. 00000800:00000200:3.0:1788972693.451033:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972693.451042:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972693.451046:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972693.451068:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972693.451073:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9d20. 00000100:00000010:3.0:1788972693.451082:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9840. 02000000:00000010:3.0:1788972693.451099:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb44548000. 00000400:00000200:3.0:1788972693.451104:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972693.451115:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78580. 00000400:00000010:3.0:1788972693.451120:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972693.451123:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108736, portal 25 00000400:00000010:3.0:1788972693.451126:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972693.451128:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179108736, offset 0 00000400:00000200:3.0:1788972693.451132:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972693.451137:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.451140:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972693.451143:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972693.451145:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972693.451148:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972693.451150:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972693.451153:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [637:637:255:4294967295] 192.168.203.124@tcp LPNI seq info [637:637:7:4294967295] 00000400:00000200:3.0:1788972693.451156:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.451161:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972693.451165:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972693.451169:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1300. 00000800:00000200:3.0:1788972693.451172:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972693.451176:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972693.451179:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972693.451183:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972693.451221:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972693.451224:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1600. 00000400:00000200:3.0:1788972693.451229:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972693.451233:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972693.451239:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972693.451241:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972693.451248:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179108864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972693.451345:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972693.451348:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1300. 00000400:00000200:3.0:1788972693.451357:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972693.451360:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972693.451363:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440 00000400:00000010:3.0:1788972693.451365:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972693.451370:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5880 x1875872179108736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972709 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972693.475380:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475391:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972693.475402:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475407:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475412:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475416:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13b80 00000400:00000200:3.0:1788972693.475422:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475427:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2095 [1] + 224 00000400:00000200:3.0:1788972693.475437:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972693.475442:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972693.475455:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5880 x1875872179108736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972709 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972693.475499:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475503:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972693.475507:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475509:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475512:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475514:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13c00 00000400:00000200:3.0:1788972693.475518:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972693.475522:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2085 [1] + 224 00000400:00000200:3.0:1788972693.475532:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972693.475536:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972693.475541:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179108864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972709 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972693.475587:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78580. 00000400:00000200:3.0:1788972693.475591:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:3.0:1788972693.475594:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972693.475602:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5880 x1875872179108736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972709 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972693.475611:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5880 x1875872179108736/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972709 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972693.475632:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789f8. 00000400:00000200:3.0:1788972693.475634:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972693.475636:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972693.475639:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179108864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972709 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972693.475657:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179108864/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972709 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972693.475675:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548000. 02000000:00000010:3.0:1788972693.475681:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1500. 00000100:00000010:3.0:1788972693.475686:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972693.475689:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb44548600. 02000000:00000010:3.0:1788972693.475703:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1700. 00000100:00000010:3.0:1788972693.475705:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 00000100:00000010:3.0:1788972694.151345:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9840. 00000100:00000010:2.0:1788972694.155946:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9d20. 00000001:02000400:3.0:1788972698.332134:0:130485:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 606: lnet_legacy2yaml: file inputs === 12:51:36 (1788972696) 00000100:00000010:0.0:1788972698.567666:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972698.567702:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb53567200. 00000100:00000010:0.0:1788972698.567756:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972698.567762:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb5381d800. 02000000:00000010:0.0:1788972698.567813:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51017200. 00000400:00000200:0.0:1788972698.567826:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972698.567831:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163738. 00000400:00000010:0.0:1788972698.567840:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788972698.567850:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109120, portal 10 00000400:00000010:0.0:1788972698.567858:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084990. 00000100:00000200:0.0:1788972698.567862:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179109120, offset 0 00000400:00000200:0.0:1788972698.567874:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972698.567896:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972698.567907:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972698.567914:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972698.567918:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972698.567923:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972698.567926:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972698.567931:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [638:638:256:4294967295] 192.168.203.124@tcp LPNI seq info [638:638:8:4294967295] 00000400:00000200:0.0:1788972698.567936:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972698.567947:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972698.567955:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972698.567961:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381dd00. 00000800:00000200:0.0:1788972698.567969:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972698.567978:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972698.567984:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381dd00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972698.568013:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972698.568027:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324120. 00000100:00000010:0.0:1788972698.568040:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324300. 02000000:00000010:0.0:1788972698.568077:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51016400. 00000400:00000200:0.0:1788972698.568100:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972698.568103:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e18. 00000400:00000010:0.0:1788972698.568107:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084bb0. 00000100:00000200:0.0:1788972698.568110:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179108992, portal 25 00000400:00000010:0.0:1788972698.568114:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084088. 00000100:00000200:0.0:1788972698.568116:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179108992, offset 0 00000400:00000200:0.0:1788972698.568121:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972698.568124:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972698.568128:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972698.568131:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972698.568134:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972698.568136:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972698.568138:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972698.568141:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [639:639:255:4294967295] 192.168.203.124@tcp LPNI seq info [639:639:7:4294967295] 00000400:00000200:0.0:1788972698.568145:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972698.568151:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972698.568155:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972698.568159:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb5381d200. 00000800:00000200:0.0:1788972698.568163:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972698.568167:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972698.568170:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb5381d200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972698.568175:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972698.568352:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972698.568358:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381dd00. 00000400:00000200:3.0:1788972698.568367:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972698.568373:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972698.568379:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084990 00000400:00000010:3.0:1788972698.568382:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084990. 00000100:00000200:3.0:1788972698.568393:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179109120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972698.568508:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972698.568511:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb5381d200. 00000400:00000200:3.0:1788972698.568514:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972698.568518:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972698.568521:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084088 00000400:00000010:3.0:1788972698.568523:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084088. 00000100:00000200:3.0:1788972698.568528:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179108992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972714 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972698.579940:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.579952:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972698.579962:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.579967:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.579974:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.579978:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13c80 00000400:00000200:3.0:1788972698.579985:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.579992:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x20ad [1] + 224 00000400:00000200:3.0:1788972698.580006:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972698.580011:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972698.580426:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179108992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972714 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972698.580471:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.580475:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972698.580479:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.580482:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.580484:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.580487:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13d00 00000400:00000200:3.0:1788972698.580491:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972698.580496:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x20a5 [1] + 224 00000400:00000200:3.0:1788972698.580505:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972698.580526:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972698.580534:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179109120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972714 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972698.580999:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e18. 00000400:00000200:0.0:1788972698.581006:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084bb0 00000400:00000010:0.0:1788972698.581009:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084bb0. 00000100:00000200:0.0:1788972698.581018:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179108992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972714 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972698.581276:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179108992/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972714 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972698.581490:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163738. 00000400:00000200:0.0:1788972698.581506:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000010:0.0:1788972698.581508:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788972698.581514:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179109120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972714 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972698.581524:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179109120/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972714 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972698.581553:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51016400. 02000000:00000010:0.0:1788972698.581561:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb53567200. 00000100:00000010:0.0:1788972698.581569:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972698.581573:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51017200. 02000000:00000010:0.0:1788972698.581576:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb5381d800. 00000100:00000010:0.0:1788972698.581579:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 00000100:00000010:2.0:1788972699.271431:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324120. 00000100:00000010:3.0:1788972699.271597:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324300. 00000100:00000010:2.0:1788972703.689179:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788972703.689197:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e900. 00000100:00000010:2.0:1788972703.689232:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411c00. 02000000:00000010:2.0:1788972703.689236:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e400. 02000000:00000010:2.0:1788972703.689265:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a200. 00000400:00000200:2.0:1788972703.689273:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972703.689277:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788972703.689283:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972703.689289:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109376, portal 10 00000400:00000010:2.0:1788972703.689296:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0330. 00000100:00000200:2.0:1788972703.689326:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179109376, offset 0 00000400:00000200:2.0:1788972703.689335:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972703.689344:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972703.689350:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972703.689355:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972703.689359:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972703.689372:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972703.689375:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972703.689379:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [640:640:256:4294967295] 192.168.203.124@tcp LPNI seq info [640:640:8:4294967295] 00000400:00000200:2.0:1788972703.689383:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972703.689392:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972703.689398:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972703.689402:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00. 00000800:00000200:2.0:1788972703.689409:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972703.689416:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972703.689420:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972703.689433:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972703.689437:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11f60. 00000100:00000010:2.0:1788972703.689445:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11720. 02000000:00000010:2.0:1788972703.689466:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a800. 00000400:00000200:2.0:1788972703.689470:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972703.689472:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788972703.689475:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972703.689477:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109248, portal 25 00000400:00000010:2.0:1788972703.689481:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0ee0. 00000100:00000200:2.0:1788972703.689483:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179109248, offset 0 00000400:00000200:2.0:1788972703.689486:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972703.689490:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972703.689493:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972703.689495:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972703.689498:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972703.689504:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972703.689506:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972703.689510:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [641:641:255:4294967295] 192.168.203.124@tcp LPNI seq info [641:641:7:4294967295] 00000400:00000200:2.0:1788972703.689514:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972703.689519:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972703.689524:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972703.689527:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e200. 00000800:00000200:2.0:1788972703.689531:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972703.689535:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972703.689537:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972703.689541:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972703.689787:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972703.689792:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00. 00000400:00000200:3.0:1788972703.689797:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972703.689801:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972703.689806:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0330 00000400:00000010:3.0:1788972703.689809:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0330. 00000100:00000200:3.0:1788972703.689817:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411c00 x1875872179109376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972703.689879:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972703.689881:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e200. 00000400:00000200:3.0:1788972703.689884:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972703.689887:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972703.689889:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0ee0 00000400:00000010:3.0:1788972703.689891:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0ee0. 00000100:00000200:3.0:1788972703.689895:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43412d80 x1875872179109248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972719 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972703.711239:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.711254:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972703.711266:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.711283:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.711290:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.711296:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13e00 00000400:00000200:3.0:1788972703.711342:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.711351:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x20b5 [1] + 224 00000400:00000200:3.0:1788972703.711368:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972703.711374:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972703.711391:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411c00 x1875872179109376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972719 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972703.714371:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.714385:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972703.714397:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.714404:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.714418:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.714423:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13d80 00000400:00000200:3.0:1788972703.714429:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972703.714436:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x20c5 [1] + 224 00000400:00000200:3.0:1788972703.714453:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972703.714458:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972703.714476:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43412d80 x1875872179109248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972719 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972703.714695:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788972703.714705:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:2.0:1788972703.714707:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972703.714721:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43412d80 x1875872179109248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972719 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972703.714732:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43412d80 x1875872179109248/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972719 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972703.714756:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788972703.714758:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:2.0:1788972703.714760:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972703.714765:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411c00 x1875872179109376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972719 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972703.714773:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411c00 x1875872179109376/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972719 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972703.714795:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a800. 02000000:00000010:2.0:1788972703.714800:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e900. 00000100:00000010:2.0:1788972703.714804:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788972703.714808:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a200. 02000000:00000010:2.0:1788972703.714819:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e400. 00000100:00000010:2.0:1788972703.714821:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411c00. 00000100:00000010:3.0:1788972704.393401:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11720. 00000100:00000010:2.0:1788972704.394435:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11f60. 00000100:00000010:2.0:1788972708.807739:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411c00. 02000000:00000010:2.0:1788972708.807771:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e400. 00000100:00000010:2.0:1788972708.807813:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43412d80. 02000000:00000010:2.0:1788972708.807819:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e600. 02000000:00000010:2.0:1788972708.807868:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c000. 00000400:00000200:2.0:1788972708.807890:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972708.807895:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788972708.807903:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0c38. 00000100:00000200:2.0:1788972708.807913:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109504, portal 25 00000400:00000010:2.0:1788972708.807921:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972708.807924:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179109504, offset 0 00000400:00000200:2.0:1788972708.807935:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972708.807946:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.807957:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972708.807963:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972708.807968:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972708.807973:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972708.807976:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972708.807980:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [642:642:256:4294967295] 192.168.203.124@tcp LPNI seq info [642:642:8:4294967295] 00000400:00000200:2.0:1788972708.807985:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.807995:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972708.808002:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972708.808011:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e900. 00000800:00000200:2.0:1788972708.808019:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972708.808029:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972708.808034:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972708.808051:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972708.808057:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11f60. 00000100:00000010:2.0:1788972708.808084:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11b40. 02000000:00000010:2.0:1788972708.808106:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124de00. 00000400:00000200:2.0:1788972708.808110:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972708.808113:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788972708.808116:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b06e8. 00000100:00000200:2.0:1788972708.808120:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109632, portal 10 00000400:00000010:2.0:1788972708.808125:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972708.808127:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179109632, offset 0 00000400:00000200:2.0:1788972708.808131:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972708.808134:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.808137:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972708.808140:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972708.808142:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972708.808145:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972708.808147:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972708.808150:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [643:643:255:4294967295] 192.168.203.124@tcp LPNI seq info [643:643:7:4294967295] 00000400:00000200:2.0:1788972708.808154:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.808158:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972708.808162:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972708.808165:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e100. 00000800:00000200:2.0:1788972708.808169:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972708.808173:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972708.808176:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972708.808182:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972708.808238:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972708.808242:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e900. 00000400:00000200:2.0:1788972708.808249:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972708.808254:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972708.808259:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:2.0:1788972708.808262:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972708.808272:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411c00 x1875872179109504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972708.808538:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972708.808541:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e100. 00000400:00000200:2.0:1788972708.808545:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972708.808548:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972708.808551:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972708.808553:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972708.808559:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43412d80 x1875872179109632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972724 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972708.812811:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.812823:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972708.812833:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.812838:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.812845:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.812849:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e13f00 00000400:00000200:2.0:1788972708.812854:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.812862:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x20e5 [1] + 224 00000400:00000200:2.0:1788972708.812900:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972708.812907:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972708.812931:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43412d80 x1875872179109632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972724 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972708.813447:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:3.0:1788972708.813454:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b06e8 00000400:00000010:3.0:1788972708.813457:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b06e8. 00000100:00000200:3.0:1788972708.813465:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43412d80 x1875872179109632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972724 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972708.813475:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43412d80 x1875872179109632/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972724 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972708.813518:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124de00. 02000000:00000010:3.0:1788972708.813527:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e600. 00000100:00000010:3.0:1788972708.813533:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43412d80. 00000400:00000200:2.0:1788972708.819257:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.819270:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972708.819280:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.819286:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.819292:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.819297:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13e80 00000400:00000200:2.0:1788972708.819332:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972708.819340:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x20d5 [1] + 224 00000400:00000200:2.0:1788972708.819358:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972708.819371:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972708.819389:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411c00 x1875872179109504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972724 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972708.822038:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:3.0:1788972708.822053:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0c38 00000400:00000010:3.0:1788972708.822056:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0c38. 00000100:00000200:3.0:1788972708.822073:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411c00 x1875872179109504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972724 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972708.822099:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411c00 x1875872179109504/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972724 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972708.822135:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c000. 02000000:00000010:3.0:1788972708.822142:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e400. 00000100:00000010:3.0:1788972708.822147:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411c00. 00000100:00000010:3.0:1788972709.511398:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11b40. 00000100:00000010:2.0:1788972709.511576:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11f60. 00000001:02000400:2.0:1788972710.185178:0:131059:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 607: lnet_legacy2yaml: options line with combined ip2nets and routes ========================================================== 12:51:48 (1788972708) 00000100:00000010:0.0:1788972713.927585:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353ea00. 02000000:00000010:0.0:1788972713.927617:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285e00. 02000000:00000010:0.0:1788972713.927696:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0200. 00000400:00000200:0.0:1788972713.927709:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972713.927714:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163738. 00000400:00000010:0.0:1788972713.927723:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084110. 00000100:00000200:0.0:1788972713.927741:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109760, portal 25 00000400:00000010:0.0:1788972713.927748:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084bb0. 00000100:00000200:0.0:1788972713.927753:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179109760, offset 0 00000400:00000200:0.0:1788972713.927764:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972713.927776:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972713.927786:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972713.927793:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972713.927798:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972713.927803:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972713.927807:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972713.927811:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [644:644:256:4294967295] 192.168.203.124@tcp LPNI seq info [644:644:8:4294967295] 00000400:00000200:0.0:1788972713.927816:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972713.927828:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972713.927857:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972713.927862:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285c00. 00000800:00000200:0.0:1788972713.927871:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972713.927880:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972713.927886:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972713.927909:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972713.927927:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324ea0. 00000100:00000010:0.0:1788972713.927940:0:100240:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb413246c0. 00000100:00000010:0.0:1788972713.927958:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353f800. 02000000:00000010:0.0:1788972713.927967:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb44d8f900. 02000000:00000010:0.0:1788972713.927993:0:100240:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0a00. 00000400:00000200:0.0:1788972713.927999:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972713.928002:0:100240:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e18. 00000400:00000010:0.0:1788972713.928006:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084cc0. 00000100:00000200:0.0:1788972713.928009:0:100240:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179109888, portal 10 00000400:00000010:0.0:1788972713.928015:0:100240:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084000. 00000100:00000200:0.0:1788972713.928018:0:100240:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179109888, offset 0 00000400:00000200:0.0:1788972713.928022:0:100240:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972713.928026:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972713.928030:0:100240:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972713.928033:0:100240:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972713.928035:0:100240:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972713.928037:0:100240:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972713.928040:0:100240:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972713.928043:0:100240:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [645:645:255:4294967295] 192.168.203.124@tcp LPNI seq info [645:645:7:4294967295] 00000400:00000200:0.0:1788972713.928047:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972713.928053:0:100240:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972713.928056:0:100240:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972713.928060:0:100240:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb44d8f400. 00000800:00000200:0.0:1788972713.928064:0:100240:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972713.928080:0:100240:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972713.928083:0:100240:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb44d8f400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972713.928089:0:100240:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972713.928387:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972713.928393:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285c00. 00000400:00000200:2.0:1788972713.928403:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972713.928408:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972713.928415:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084bb0 00000400:00000010:2.0:1788972713.928418:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084bb0. 00000100:00000200:2.0:1788972713.928430:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353ea00 x1875872179109760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972713.928522:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972713.928525:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb44d8f400. 00000400:00000200:2.0:1788972713.928528:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972713.928531:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972713.928534:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084000 00000400:00000010:2.0:1788972713.928535:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084000. 00000100:00000200:2.0:1788972713.928540:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353f800 x1875872179109888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972729 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972713.947400:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947413:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972713.947424:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947430:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947436:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947441:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e13f80 00000400:00000200:3.0:1788972713.947447:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947456:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x20f5 [1] + 224 00000400:00000200:3.0:1788972713.947470:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972713.947475:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972713.947495:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353ea00 x1875872179109760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972729 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972713.947560:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947564:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972713.947570:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947573:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947577:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947580:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14000 00000400:00000200:3.0:1788972713.947584:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972713.947590:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x20fd [1] + 224 00000400:00000010:1.0:1788972713.947602:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163738. 00000400:00000200:1.0:1788972713.947611:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084110 00000400:00000200:3.0:1788972713.947613:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:1.0:1788972713.947615:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084110. 00000400:00000200:3.0:1788972713.947621:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:1.0:1788972713.947628:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353ea00 x1875872179109760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972729 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972713.947647:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353ea00 x1875872179109760/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972729 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972713.947656:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353f800 x1875872179109888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972729 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972713.947687:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0200. 02000000:00000010:1.0:1788972713.947697:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285e00. 00000100:00000010:1.0:1788972713.947703:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353ea00. 00000400:00000010:1.0:1788972713.947711:0:100240:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e18. 00000400:00000200:1.0:1788972713.947713:0:100240:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084cc0 00000400:00000010:1.0:1788972713.947715:0:100240:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084cc0. 00000100:00000200:1.0:1788972713.947719:0:100240:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353f800 x1875872179109888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972729 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972713.947727:0:100240:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353f800 x1875872179109888/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972729 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972713.947817:0:100240:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0a00. 02000000:00000010:1.0:1788972713.947822:0:100240:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb44d8f900. 00000100:00000010:1.0:1788972713.947832:0:100240:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353f800. 00000100:00000010:2.0:1788972714.634626:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324ea0. 00000100:00000010:3.0:1788972714.635576:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb413246c0. 00000001:02000400:3.0:1788972717.600270:0:131631:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 608: lnet_legacy2yaml: error on invalid rule (missing IP pattern) ========================================================== 12:51:56 (1788972716) 00000100:00000010:2.0:1788972719.051337:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410a80. 02000000:00000010:2.0:1788972719.051358:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ee00. 00000100:00000010:2.0:1788972719.051390:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788972719.051395:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e900. 02000000:00000010:2.0:1788972719.051427:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0600. 00000400:00000200:2.0:1788972719.051435:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972719.051439:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788972719.051445:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972719.051458:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110016, portal 25 00000400:00000010:2.0:1788972719.051463:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972719.051466:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179110016, offset 0 00000400:00000200:2.0:1788972719.051475:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972719.051483:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.051490:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972719.051495:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972719.051499:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972719.051503:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972719.051506:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972719.051510:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [646:646:256:4294967295] 192.168.203.124@tcp LPNI seq info [646:646:8:4294967295] 00000400:00000200:2.0:1788972719.051526:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.051535:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972719.051541:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972719.051546:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e200. 00000800:00000200:2.0:1788972719.051552:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972719.051558:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972719.051563:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972719.051576:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972719.051581:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f115a0. 00000100:00000010:2.0:1788972719.051593:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11f60. 02000000:00000010:2.0:1788972719.051613:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0200. 00000400:00000200:2.0:1788972719.051617:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972719.051620:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788972719.051623:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972719.051626:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110144, portal 10 00000400:00000010:2.0:1788972719.051630:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788972719.051632:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179110144, offset 0 00000400:00000200:2.0:1788972719.051636:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972719.051639:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.051642:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972719.051651:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972719.051653:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972719.051655:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972719.051657:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972719.051660:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [647:647:255:4294967295] 192.168.203.124@tcp LPNI seq info [647:647:7:4294967295] 00000400:00000200:2.0:1788972719.051664:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.051670:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972719.051674:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972719.051677:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e400. 00000800:00000200:2.0:1788972719.051681:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972719.051685:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972719.051687:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972719.051693:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972719.051735:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972719.051739:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e200. 00000400:00000200:2.0:1788972719.051742:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972719.051747:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972719.051751:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:2.0:1788972719.051753:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972719.051761:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410a80 x1875872179110016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972719.051828:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972719.051831:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e400. 00000400:00000200:2.0:1788972719.051834:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972719.051838:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972719.051840:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28 00000400:00000010:2.0:1788972719.051842:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788972719.051846:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179110144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972735 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972719.056352:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.056370:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972719.056379:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.056384:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.056389:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.056393:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14080 00000400:00000200:2.0:1788972719.056398:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.056404:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2105 [1] + 224 00000400:00000200:2.0:1788972719.056418:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972719.056423:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972719.056436:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410a80 x1875872179110016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972735 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972719.056479:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:2.0:1788972719.056484:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972719.056487:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972719.056493:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410a80 x1875872179110016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972735 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972719.056502:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410a80 x1875872179110016/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972735 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972719.056543:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0600. 02000000:00000010:2.0:1788972719.056549:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ee00. 00000100:00000010:2.0:1788972719.056553:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410a80. 00000400:00000200:2.0:1788972719.063684:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.063694:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972719.063703:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.063708:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.063713:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.063717:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14100 00000400:00000200:2.0:1788972719.063723:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972719.063729:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2115 [1] + 224 00000400:00000200:2.0:1788972719.063744:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972719.063755:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972719.063769:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179110144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972735 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972719.063811:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:2.0:1788972719.063815:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:2.0:1788972719.063818:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972719.063824:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179110144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972735 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972719.063833:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179110144/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972735 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972719.063865:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0200. 02000000:00000010:2.0:1788972719.063871:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e900. 00000100:00000010:2.0:1788972719.063876:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000100:00000010:3.0:1788972719.751863:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11f60. 00000100:00000010:2.0:1788972719.754909:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f115a0. 00000001:02000400:0.0:1788972723.782254:0:132204:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 609: lnet_legacy2yaml: error on no input ========================================================== 12:52:02 (1788972722) 00000100:00000010:2.0:1788972724.167555:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411f80. 02000000:00000010:2.0:1788972724.167578:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ee00. 00000100:00000010:2.0:1788972724.167623:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410a80. 02000000:00000010:2.0:1788972724.167629:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e600. 02000000:00000010:2.0:1788972724.167662:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa800. 00000400:00000200:2.0:1788972724.167673:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972724.167677:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788972724.167685:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0a18. 00000100:00000200:2.0:1788972724.167693:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110400, portal 10 02000000:00000010:3.0:1788972724.167695:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69600. 00000400:00000010:2.0:1788972724.167702:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000400:00000200:3.0:1788972724.167706:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000200:2.0:1788972724.167708:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179110400, offset 0 00000400:00000010:3.0:1788972724.167712:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789f8. 00000400:00000200:2.0:1788972724.167719:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000010:3.0:1788972724.167728:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000400:00000200:2.0:1788972724.167729:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000200:3.0:1788972724.167736:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110272, portal 25 00000400:00000200:2.0:1788972724.167740:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000010:3.0:1788972724.167742:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081880. 00000100:00000200:3.0:1788972724.167747:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179110272, offset 0 00000400:00000200:2.0:1788972724.167748:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972724.167755:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972724.167756:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972724.167762:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972724.167774:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972724.167777:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [648:648:256:4294967295] 192.168.203.124@tcp LPNI seq info [648:648:8:4294967295] 00000400:00000200:2.0:1788972724.167782:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.167792:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972724.167798:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972724.167800:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000010:2.0:1788972724.167805:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e900. 00000400:00000200:3.0:1788972724.167808:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972724.167814:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:2.0:1788972724.167815:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000400:00000200:3.0:1788972724.167819:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972724.167824:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972724.167827:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972724.167830:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [649:649:255:4294967295] 192.168.203.124@tcp LPNI seq info [649:649:7:4294967295] 00000400:00000200:3.0:1788972724.167835:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972724.167840:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972724.167846:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972724.167848:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972724.167855:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972724.167860:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1f00. 00000400:00000200:2.0:1788972724.167864:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972724.167868:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000100:00000010:2.0:1788972724.167872:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11ba0. 00000800:00000200:3.0:1788972724.167877:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972724.167882:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1f00 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:2.0:1788972724.167884:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11a20. 00000400:00000200:3.0:1788972724.167889:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972724.167939:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972724.167942:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e900. 00000400:00000200:2.0:1788972724.167948:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972724.167953:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972724.167958:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972724.167960:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972724.167967:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410a80 x1875872179110400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972724.168165:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972724.168168:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1f00. 00000400:00000200:2.0:1788972724.168172:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972724.168176:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972724.168179:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081880 00000400:00000010:2.0:1788972724.168181:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081880. 00000100:00000200:2.0:1788972724.168187:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411f80 x1875872179110272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972740 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972724.181908:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.181917:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972724.181937:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.181943:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.181947:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.181952:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14180 00000400:00000200:2.0:1788972724.181957:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.181964:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2135 [1] + 224 00000400:00000200:2.0:1788972724.181979:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972724.181985:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972724.182009:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411f80 x1875872179110272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972740 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972724.182073:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789f8. 00000400:00000200:3.0:1788972724.182086:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972724.182088:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972724.182095:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411f80 x1875872179110272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972740 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972724.182106:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411f80 x1875872179110272/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972740 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972724.182133:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69600. 02000000:00000010:3.0:1788972724.182140:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ee00. 00000100:00000010:3.0:1788972724.182146:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411f80. 00000400:00000200:2.0:1788972724.185379:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.185388:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972724.185397:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.185402:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.185407:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.185410:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14200 00000400:00000200:2.0:1788972724.185416:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972724.185421:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2125 [1] + 224 00000400:00000200:2.0:1788972724.185435:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972724.185439:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972724.185452:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410a80 x1875872179110400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972740 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972724.185556:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:3.0:1788972724.185562:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0a18 00000400:00000010:3.0:1788972724.185565:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0a18. 00000100:00000200:3.0:1788972724.185572:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410a80 x1875872179110400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972740 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972724.185583:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410a80 x1875872179110400/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972740 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972724.185614:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa800. 02000000:00000010:3.0:1788972724.185620:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e600. 00000100:00000010:3.0:1788972724.185625:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410a80. 00000100:00000010:3.0:1788972724.871372:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11a20. 00000100:00000010:2.0:1788972724.871410:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11ba0. 00000100:00000010:0.0:1788972729.288143:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788972729.288161:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb48503e00. 00000100:00000010:0.0:1788972729.288190:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972729.288194:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb48503f00. 02000000:00000010:0.0:1788972729.288223:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c400. 00000400:00000200:0.0:1788972729.288232:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972729.288236:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e18. 00000400:00000010:0.0:1788972729.288243:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084440. 00000100:00000200:0.0:1788972729.288250:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110528, portal 25 00000400:00000010:0.0:1788972729.288255:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480843b8. 00000100:00000200:0.0:1788972729.288258:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179110528, offset 0 00000400:00000200:0.0:1788972729.288266:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972729.288274:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972729.288282:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972729.288287:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972729.288291:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972729.288295:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972729.288317:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972729.288320:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [650:650:256:4294967295] 192.168.203.124@tcp LPNI seq info [650:650:8:4294967295] 00000400:00000200:0.0:1788972729.288325:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972729.288335:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972729.288341:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972729.288346:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb48503b00. 00000800:00000200:0.0:1788972729.288353:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972729.288360:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972729.288364:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb48503b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972729.288384:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972729.288388:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324a80. 00000100:00000010:0.0:1788972729.288398:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb413242a0. 02000000:00000010:0.0:1788972729.288417:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124ca00. 00000400:00000200:0.0:1788972729.288422:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972729.288424:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163738. 00000400:00000010:0.0:1788972729.288427:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084908. 00000100:00000200:0.0:1788972729.288430:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110656, portal 10 00000400:00000010:0.0:1788972729.288435:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480847f8. 00000100:00000200:0.0:1788972729.288437:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179110656, offset 0 00000400:00000200:0.0:1788972729.288451:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972729.288455:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972729.288458:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972729.288460:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972729.288462:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972729.288464:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972729.288466:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972729.288468:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [651:651:255:4294967295] 192.168.203.124@tcp LPNI seq info [651:651:7:4294967295] 00000400:00000200:0.0:1788972729.288472:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972729.288476:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972729.288480:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972729.288483:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb48503100. 00000800:00000200:0.0:1788972729.288486:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972729.288489:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972729.288492:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb48503100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972729.288497:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972729.288630:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972729.288635:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb48503b00. 00000400:00000200:2.0:1788972729.288640:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972729.288644:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972729.288654:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480843b8 00000400:00000010:2.0:1788972729.288657:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480843b8. 00000100:00000200:2.0:1788972729.288665:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179110528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972729.288747:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972729.288750:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb48503100. 00000400:00000200:2.0:1788972729.288753:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972729.288757:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972729.288760:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480847f8 00000400:00000010:2.0:1788972729.288762:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480847f8. 00000100:00000200:2.0:1788972729.288767:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179110656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972745 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972729.305424:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305433:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972729.305456:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305461:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305466:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305470:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14300 00000400:00000200:2.0:1788972729.305475:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305482:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x214d [1] + 224 00000400:00000200:2.0:1788972729.305491:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972729.305496:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972729.305509:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179110656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972745 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972729.305577:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305581:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972729.305585:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305588:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305591:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305594:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14280 00000400:00000200:2.0:1788972729.305597:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972729.305601:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2145 [1] + 224 00000400:00000200:2.0:1788972729.305609:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972729.305619:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972729.305626:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179110528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972745 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972729.306191:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163738. 00000400:00000200:0.0:1788972729.306198:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084908 00000400:00000010:0.0:1788972729.306200:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084908. 00000100:00000200:0.0:1788972729.306208:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179110656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972745 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972729.306328:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179110656/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972745 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972729.306358:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e18. 00000400:00000200:0.0:1788972729.306362:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084440 00000400:00000010:0.0:1788972729.306365:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084440. 00000100:00000200:0.0:1788972729.306370:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179110528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972745 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972729.306379:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179110528/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972745 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972729.306394:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124ca00. 02000000:00000010:0.0:1788972729.306400:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb48503f00. 00000100:00000010:0.0:1788972729.306407:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972729.306411:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c400. 02000000:00000010:0.0:1788972729.306413:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb48503e00. 00000100:00000010:0.0:1788972729.306415:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000100:00000010:2.0:1788972729.991869:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324a80. 00000100:00000010:3.0:1788972729.992491:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb413242a0. 00000001:02000400:1.0:1788972731.631274:0:132776:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 610: lnet_legacy2yaml: multiple files and inline, check order ========================================================== 12:52:09 (1788972729) 00000100:00000010:3.0:1788972734.409496:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788972734.409518:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1f00. 00000100:00000010:3.0:1788972734.409552:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972734.409558:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1600. 02000000:00000010:3.0:1788972734.409589:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3c00. 00000400:00000200:3.0:1788972734.409599:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972734.409604:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789f8. 00000400:00000010:3.0:1788972734.409610:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972734.409616:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110912, portal 10 00000400:00000010:3.0:1788972734.409631:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972734.409634:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179110912, offset 0 00000400:00000200:3.0:1788972734.409644:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972734.409653:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972734.409660:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972734.409665:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972734.409670:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972734.409673:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972734.409676:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972734.409680:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [652:652:256:4294967295] 192.168.203.124@tcp LPNI seq info [652:652:8:4294967295] 00000400:00000200:3.0:1788972734.409684:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972734.409693:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972734.409699:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972734.409703:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1b00. 00000800:00000200:3.0:1788972734.409710:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972734.409717:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972734.409722:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972734.409737:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972734.409743:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9720. 00000100:00000010:3.0:1788972734.409752:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9180. 02000000:00000010:3.0:1788972734.409770:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f3200. 00000400:00000200:3.0:1788972734.409775:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972734.409777:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78580. 00000400:00000010:3.0:1788972734.409780:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972734.409783:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179110784, portal 25 00000400:00000010:3.0:1788972734.409786:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788972734.409788:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179110784, offset 0 00000400:00000200:3.0:1788972734.409793:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972734.409796:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972734.409799:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972734.409802:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972734.409810:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972734.409812:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972734.409815:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972734.409817:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [653:653:255:4294967295] 192.168.203.124@tcp LPNI seq info [653:653:7:4294967295] 00000400:00000200:3.0:1788972734.409821:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972734.409827:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972734.409831:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972734.409835:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100. 00000800:00000200:3.0:1788972734.409838:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972734.409842:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972734.409845:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972734.409849:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972734.409905:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972734.409910:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1b00. 00000400:00000200:3.0:1788972734.409916:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972734.409920:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972734.409925:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440 00000400:00000010:3.0:1788972734.409927:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972734.409935:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5880 x1875872179110912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972734.410013:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972734.410016:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100. 00000400:00000200:3.0:1788972734.410018:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972734.410022:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972734.410024:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081c38 00000400:00000010:3.0:1788972734.410026:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788972734.410030:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179110784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972750 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972734.413387:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.413397:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972734.413417:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.413421:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.413426:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.413430:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14380 00000400:00000200:2.0:1788972734.413435:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.413440:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2165 [1] + 224 00000400:00000200:2.0:1788972734.413454:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972734.413458:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972734.413470:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179110784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972750 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972734.418431:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78580. 00000400:00000200:3.0:1788972734.418442:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972734.418444:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972734.418465:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179110784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972750 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972734.418477:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179110784/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972750 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972734.418502:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3200. 02000000:00000010:3.0:1788972734.418509:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1f00. 00000100:00000010:3.0:1788972734.418516:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 00000400:00000200:2.0:1788972734.418587:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.418593:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972734.418600:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.418605:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.418609:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.418614:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14400 00000400:00000200:2.0:1788972734.418618:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972734.418623:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2155 [1] + 224 00000400:00000200:2.0:1788972734.418636:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972734.418641:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972734.418651:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5880 x1875872179110912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972750 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972734.418715:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789f8. 00000400:00000200:3.0:1788972734.418719:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:3.0:1788972734.418720:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972734.418725:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5880 x1875872179110912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972750 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972734.418733:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5880 x1875872179110912/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972750 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972734.418751:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f3c00. 02000000:00000010:3.0:1788972734.418755:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1600. 00000100:00000010:3.0:1788972734.418758:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880. 00000100:00000010:3.0:1788972735.115166:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9180. 00000100:00000010:2.0:1788972735.120137:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9720. 00000100:00000010:2.0:1788972739.529651:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972739.529680:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e100. 00000100:00000010:2.0:1788972739.529718:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972739.529724:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e400. 02000000:00000010:2.0:1788972739.529777:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69c00. 00000400:00000200:2.0:1788972739.529790:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972739.529794:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788972739.529802:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972739.529811:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111040, portal 25 00000400:00000010:2.0:1788972739.529818:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788972739.529822:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179111040, offset 0 00000400:00000200:2.0:1788972739.529834:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972739.529845:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.529856:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972739.529862:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972739.529868:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972739.529873:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972739.529877:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972739.529881:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [654:654:256:4294967295] 192.168.203.124@tcp LPNI seq info [654:654:8:4294967295] 00000400:00000200:2.0:1788972739.529886:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.529896:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972739.529904:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972739.529910:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e200. 00000800:00000200:2.0:1788972739.529918:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972739.529928:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972739.529933:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972739.529949:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972739.529955:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11060. 00000100:00000010:2.0:1788972739.529968:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f112a0. 02000000:00000010:2.0:1788972739.529993:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69200. 00000400:00000200:2.0:1788972739.529998:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972739.530010:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c60. 00000400:00000010:2.0:1788972739.530014:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b02a8. 00000100:00000200:2.0:1788972739.530017:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111168, portal 10 00000400:00000010:2.0:1788972739.530024:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972739.530026:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179111168, offset 0 00000400:00000200:2.0:1788972739.530031:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972739.530035:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.530039:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972739.530041:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972739.530044:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972739.530046:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972739.530048:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972739.530051:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [655:655:255:4294967295] 192.168.203.124@tcp LPNI seq info [655:655:7:4294967295] 00000400:00000200:2.0:1788972739.530055:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.530060:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972739.530065:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972739.530069:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e600. 00000800:00000200:2.0:1788972739.530073:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972739.530077:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972739.530080:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972739.530086:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972739.530162:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972739.530167:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e200. 00000400:00000200:2.0:1788972739.530173:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972739.530185:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972739.530193:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28 00000400:00000010:2.0:1788972739.530195:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788972739.530206:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179111040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972739.530336:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972739.530339:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e600. 00000400:00000200:2.0:1788972739.530349:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972739.530353:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972739.530356:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:2.0:1788972739.530358:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972739.530364:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179111168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972755 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972739.537991:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.538004:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972739.538014:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.538020:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.538026:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.538030:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14500 00000400:00000200:2.0:1788972739.538048:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.538056:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2185 [1] + 224 00000400:00000200:2.0:1788972739.538076:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972739.538080:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972739.538097:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179111168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972755 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972739.538580:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c60. 00000400:00000200:3.0:1788972739.538586:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b02a8 00000400:00000010:3.0:1788972739.538589:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b02a8. 00000100:00000200:3.0:1788972739.538606:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179111168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972755 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972739.538616:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179111168/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972755 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972739.538694:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69200. 02000000:00000010:3.0:1788972739.538702:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e400. 00000100:00000010:3.0:1788972739.538708:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 00000400:00000200:2.0:1788972739.539049:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.539055:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972739.539062:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.539066:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.539070:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.539074:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14480 00000400:00000200:2.0:1788972739.539078:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972739.539084:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2175 [1] + 224 00000400:00000200:2.0:1788972739.539106:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972739.539118:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972739.539127:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179111040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972755 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972739.539170:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:2.0:1788972739.539173:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:2.0:1788972739.539175:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972739.539534:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179111040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972755 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972739.539543:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179111040/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972755 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972739.539563:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69c00. 02000000:00000010:2.0:1788972739.539568:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e100. 00000100:00000010:2.0:1788972739.539572:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 00000001:02000400:3.0:1788972740.065880:0:133349:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 611: lnet_legacy2yaml: invalid hopcount ========================================================== 12:52:18 (1788972738) 00000100:00000010:2.0:1788972740.232622:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11060. 00000100:00000010:3.0:1788972740.233903:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f112a0. 00000100:00000010:3.0:1788972744.649839:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5880. 02000000:00000010:3.0:1788972744.649860:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1100. 00000100:00000010:3.0:1788972744.649889:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788972744.649894:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1b00. 02000000:00000010:3.0:1788972744.649929:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69400. 00000400:00000200:3.0:1788972744.649938:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972744.649942:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e789f8. 00000400:00000010:3.0:1788972744.649949:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081908. 00000100:00000200:3.0:1788972744.649956:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111296, portal 25 00000400:00000010:3.0:1788972744.649962:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972744.649964:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179111296, offset 0 00000400:00000200:3.0:1788972744.649973:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972744.649981:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.649988:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972744.649992:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972744.649996:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972744.649999:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972744.650002:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972744.650005:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [656:656:256:4294967295] 192.168.203.124@tcp LPNI seq info [656:656:8:4294967295] 00000400:00000200:3.0:1788972744.650009:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.650017:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972744.650023:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972744.650029:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1500. 00000800:00000200:3.0:1788972744.650035:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972744.650042:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972744.650046:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972744.650059:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972744.650065:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9000. 00000100:00000010:3.0:1788972744.650075:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9de0. 02000000:00000010:3.0:1788972744.650101:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb49d69a00. 00000400:00000200:3.0:1788972744.650106:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972744.650108:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78580. 00000400:00000010:3.0:1788972744.650111:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788972744.650114:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111424, portal 10 00000400:00000010:3.0:1788972744.650119:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972744.650120:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179111424, offset 0 00000400:00000200:3.0:1788972744.650125:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972744.650128:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.650131:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972744.650134:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972744.650136:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972744.650138:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972744.650140:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972744.650142:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [657:657:255:4294967295] 192.168.203.124@tcp LPNI seq info [657:657:7:4294967295] 00000400:00000200:3.0:1788972744.650146:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.650151:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972744.650155:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972744.650159:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1d00. 00000800:00000200:3.0:1788972744.650163:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972744.650166:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972744.650169:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972744.650175:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972744.650216:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972744.650220:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1500. 00000400:00000200:3.0:1788972744.650224:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972744.650229:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972744.650234:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:3.0:1788972744.650236:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972744.650244:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5880 x1875872179111296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972744.651817:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972744.651824:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1d00. 00000400:00000200:3.0:1788972744.651829:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972744.651835:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972744.651840:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440 00000400:00000010:3.0:1788972744.651842:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972744.651849:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179111424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972760 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972744.660022:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972744.660032:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972744.660040:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972744.660044:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972744.660049:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972744.660052:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14580 00000400:00000200:2.0:1788972744.660058:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972744.660064:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2195 [1] + 224 00000400:00000200:2.0:1788972744.660078:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972744.660083:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972744.660097:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5880 x1875872179111296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972760 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972744.660141:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e789f8. 00000400:00000200:2.0:1788972744.660146:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081908 00000400:00000010:2.0:1788972744.660149:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081908. 00000100:00000200:2.0:1788972744.660154:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5880 x1875872179111296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972760 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972744.660163:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5880 x1875872179111296/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972760 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972744.660201:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69400. 02000000:00000010:2.0:1788972744.660207:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1100. 00000100:00000010:2.0:1788972744.660212:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5880. 00000400:00000200:3.0:1788972744.667983:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.667994:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972744.668001:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.668006:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.668010:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.668014:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14600 00000400:00000200:3.0:1788972744.668018:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972744.668023:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x21a5 [1] + 224 00000400:00000200:3.0:1788972744.668035:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972744.668040:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972744.668053:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179111424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972760 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972744.668101:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78580. 00000400:00000200:3.0:1788972744.668106:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081c38 00000400:00000010:3.0:1788972744.668108:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788972744.668115:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179111424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972760 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972744.668124:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179111424/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972760 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972744.668154:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb49d69a00. 02000000:00000010:3.0:1788972744.668171:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1b00. 00000100:00000010:3.0:1788972744.668176:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 00000100:00000010:3.0:1788972745.352629:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9de0. 00000100:00000010:2.0:1788972745.352911:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9000. 00000001:02000400:3.0:1788972749.596279:0:133920:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 612: lnet_legacy2yaml: malformed IP pattern ========================================================== 12:52:27 (1788972747) 00000100:00000010:2.0:1788972749.776677:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972749.776700:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e600. 00000100:00000010:2.0:1788972749.776738:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972749.776744:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e100. 02000000:00000010:2.0:1788972749.776775:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a600. 00000400:00000200:2.0:1788972749.776786:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972749.776790:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b33c8. 00000400:00000010:2.0:1788972749.776797:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b04c8. 00000100:00000200:2.0:1788972749.776805:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111680, portal 10 00000400:00000010:2.0:1788972749.776811:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972749.776814:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179111680, offset 0 00000400:00000200:2.0:1788972749.776822:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972749.776830:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972749.776838:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972749.776843:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972749.776847:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972749.776850:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972749.776853:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972749.776857:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [658:658:256:4294967295] 192.168.203.124@tcp LPNI seq info [658:658:8:4294967295] 00000400:00000200:2.0:1788972749.776861:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972749.776870:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972749.776877:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972749.776882:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e200. 00000800:00000200:2.0:1788972749.776889:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972749.776896:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972749.776910:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972749.776926:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972749.776931:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f115a0. 00000100:00000010:2.0:1788972749.776942:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11600. 02000000:00000010:2.0:1788972749.776961:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4350a400. 00000400:00000200:2.0:1788972749.776965:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972749.776968:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3c60. 00000400:00000010:2.0:1788972749.776972:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0b28. 00000100:00000200:2.0:1788972749.776975:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111552, portal 25 00000400:00000010:2.0:1788972749.776978:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972749.776980:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179111552, offset 0 00000400:00000200:2.0:1788972749.776984:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972749.776988:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972749.776992:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972749.776994:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972749.776997:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972749.776999:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972749.777001:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972749.777004:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [659:659:255:4294967295] 192.168.203.124@tcp LPNI seq info [659:659:7:4294967295] 00000400:00000200:2.0:1788972749.777008:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972749.777014:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972749.777019:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972749.777021:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00. 00000800:00000200:2.0:1788972749.777025:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972749.777029:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972749.777032:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972749.777036:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972749.777076:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972749.777079:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e200. 00000400:00000200:2.0:1788972749.777083:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972749.777088:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972749.777097:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:2.0:1788972749.777100:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972749.777108:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179111680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972749.777185:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972749.777188:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00. 00000400:00000200:2.0:1788972749.777191:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972749.777202:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972749.777205:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:2.0:1788972749.777206:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972749.777211:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179111552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972765 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972749.803759:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803769:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972749.803777:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803782:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803787:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803791:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14700 00000400:00000200:3.0:1788972749.803796:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803802:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x21b5 [1] + 224 00000400:00000200:3.0:1788972749.803814:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972749.803819:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972749.803832:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179111680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972765 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972749.803862:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803865:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972749.803870:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803872:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803875:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803878:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14680 00000400:00000200:3.0:1788972749.803881:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972749.803885:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x21c5 [1] + 224 00000400:00000200:3.0:1788972749.803893:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972749.803896:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972749.803901:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179111552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972765 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972749.803931:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3c60. 00000400:00000200:3.0:1788972749.803944:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0b28 00000400:00000010:3.0:1788972749.803946:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0b28. 00000100:00000200:3.0:1788972749.803952:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179111552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972765 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972749.803959:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179111552/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972765 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972749.803979:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b33c8. 00000400:00000200:3.0:1788972749.803982:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b04c8 00000400:00000010:3.0:1788972749.803984:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b04c8. 00000100:00000200:3.0:1788972749.803988:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179111680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972765 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972749.803995:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179111680/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972765 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972749.804014:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a400. 02000000:00000010:3.0:1788972749.804018:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e600. 00000100:00000010:3.0:1788972749.804023:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 02000000:00000010:3.0:1788972749.804026:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4350a600. 02000000:00000010:3.0:1788972749.804029:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e100. 00000100:00000010:3.0:1788972749.804031:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 00000100:00000010:2.0:1788972750.471402:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f115a0. 00000100:00000010:3.0:1788972750.472242:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11600. 00000100:00000010:2.0:1788972754.892288:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788972754.892330:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604ee00. 00000100:00000010:2.0:1788972754.892375:0:119065:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972754.892381:0:119065:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e200. 02000000:00000010:2.0:1788972754.894622:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c200. 00000400:00000200:2.0:1788972754.894643:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972754.894651:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788972754.894660:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972754.894670:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111808, portal 25 00000400:00000010:2.0:1788972754.894677:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972754.894681:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179111808, offset 0 00000400:00000200:2.0:1788972754.894692:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972754.894702:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972754.894712:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972754.894719:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972754.894724:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972754.894729:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972754.894731:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972754.894735:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [660:660:256:4294967295] 192.168.203.124@tcp LPNI seq info [660:660:8:4294967295] 00000400:00000200:2.0:1788972754.894750:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972754.894761:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972754.894767:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972754.894773:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e100. 00000800:00000200:2.0:1788972754.894781:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972754.894790:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972754.894795:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972754.895201:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972754.895208:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f115a0. 00000100:00000010:2.0:1788972754.895221:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11540. 02000000:00000010:2.0:1788972754.895248:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5389c800. 00000400:00000200:2.0:1788972754.895252:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972754.895255:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788972754.895258:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972754.895271:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179111936, portal 10 00000400:00000010:2.0:1788972754.895277:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972754.895279:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179111936, offset 0 00000400:00000200:2.0:1788972754.895284:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972754.895288:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972754.895291:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972754.895294:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972754.895296:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972754.895321:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972754.895323:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972754.895326:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [661:661:255:4294967295] 192.168.203.124@tcp LPNI seq info [661:661:7:4294967295] 00000400:00000200:2.0:1788972754.895330:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972754.895336:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972754.895340:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972754.895344:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e600. 00000800:00000200:2.0:1788972754.895349:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972754.895359:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972754.895362:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972754.895370:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972754.895443:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972754.895448:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e100. 00000400:00000200:2.0:1788972754.895453:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972754.895459:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972754.895466:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:2.0:1788972754.895469:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972754.895482:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179111808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972754.895582:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972754.895585:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e600. 00000400:00000200:2.0:1788972754.895588:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972754.895591:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972754.895594:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972754.895596:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972754.895601:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179111936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972770 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972754.900869:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.900882:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972754.900893:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.900899:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.900906:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.900911:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14780 00000400:00000200:3.0:1788972754.900917:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.900925:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x21d5 [1] + 224 00000400:00000200:3.0:1788972754.900939:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972754.900944:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972754.900972:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179111808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972770 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972754.901005:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.901009:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972754.901013:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.901016:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.901019:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.901022:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14800 00000400:00000200:3.0:1788972754.901026:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972754.901030:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x21e5 [1] + 224 00000400:00000200:3.0:1788972754.901038:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972754.901041:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972754.901048:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179111936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972770 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972754.901104:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788972754.901112:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972754.901115:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972754.901122:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179111936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972770 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972754.901132:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179111936/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972770 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972754.901169:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:2.0:1788972754.901172:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:2.0:1788972754.901174:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972754.901179:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179111808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972770 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972754.901201:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179111808/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972770 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972754.901221:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c800. 02000000:00000010:2.0:1788972754.901231:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e200. 00000100:00000010:2.0:1788972754.901238:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972754.901242:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5389c200. 02000000:00000010:2.0:1788972754.901245:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604ee00. 00000100:00000010:2.0:1788972754.901247:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 00000100:00000010:3.0:1788972755.591370:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11540. 00000100:00000010:2.0:1788972755.591854:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f115a0. 00000001:02000400:0.0:1788972759.385780:0:134492:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 613: lnet_legacy2yaml: multiple IP patterns in single rule ========================================================== 12:52:37 (1788972757) 00000100:00000010:1.0:1788972760.011791:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4a80. 02000000:00000010:1.0:1788972760.011827:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972760.011870:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb5500. 02000000:00000010:1.0:1788972760.011876:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55000. 02000000:00000010:1.0:1788972760.011926:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab000. 00000400:00000200:1.0:1788972760.011938:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972760.011943:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1c08. 00000400:00000010:1.0:1788972760.011952:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972760.011961:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112064, portal 25 00000400:00000010:1.0:1788972760.011967:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588330. 00000100:00000200:1.0:1788972760.011972:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179112064, offset 0 00000400:00000200:1.0:1788972760.011983:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972760.011993:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972760.012003:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972760.012009:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972760.012014:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972760.012028:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972760.012031:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972760.012035:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [662:662:256:4294967295] 192.168.203.124@tcp LPNI seq info [662:662:8:4294967295] 00000400:00000200:1.0:1788972760.012040:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972760.012051:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972760.012057:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972760.012063:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55800. 00000800:00000200:1.0:1788972760.012071:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972760.012079:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972760.012084:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972760.012095:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972760.012101:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4427ea20. 00000100:00000010:1.0:1788972760.012120:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4427e4e0. 02000000:00000010:1.0:1788972760.012141:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459abc00. 00000400:00000200:1.0:1788972760.012146:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972760.012149:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972760.012152:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788972760.012155:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112192, portal 10 00000400:00000010:1.0:1788972760.012161:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588aa0. 00000100:00000200:1.0:1788972760.012163:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179112192, offset 0 00000400:00000200:1.0:1788972760.012167:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972760.012170:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972760.012174:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972760.012176:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972760.012179:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972760.012181:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972760.012183:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972760.012186:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [663:663:255:4294967295] 192.168.203.124@tcp LPNI seq info [663:663:7:4294967295] 00000400:00000200:1.0:1788972760.012189:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972760.012195:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972760.012200:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972760.012203:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55a00. 00000800:00000200:1.0:1788972760.012206:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972760.012210:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972760.012212:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55a00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972760.012218:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972760.013110:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972760.013117:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55800. 00000400:00000200:2.0:1788972760.013123:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972760.013132:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972760.013139:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588330 00000400:00000010:2.0:1788972760.013142:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588330. 00000100:00000200:2.0:1788972760.013153:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4a80 x1875872179112064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972760.013260:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972760.013263:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55a00. 00000400:00000200:2.0:1788972760.013267:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972760.013271:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972760.013274:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588aa0 00000400:00000010:2.0:1788972760.013276:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588aa0. 00000100:00000200:2.0:1788972760.013281:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb5500 x1875872179112192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972776 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972760.018786:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018798:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972760.018812:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018818:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018826:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018839:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14880 00000400:00000200:2.0:1788972760.018847:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018854:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x21f5 [1] + 224 00000400:00000200:2.0:1788972760.018870:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972760.018876:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972760.018905:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4a80 x1875872179112064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972776 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972760.018936:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018941:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972760.018946:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018949:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018952:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018955:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14900 00000400:00000200:2.0:1788972760.018960:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972760.018965:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x21fd [1] + 224 00000400:00000200:2.0:1788972760.018976:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972760.018979:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972760.018986:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb5500 x1875872179112192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972776 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972760.019468:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972760.019476:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:1.0:1788972760.019479:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788972760.019488:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb5500 x1875872179112192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972776 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972760.019500:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb5500 x1875872179112192/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972776 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972760.019568:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1c08. 00000400:00000200:1.0:1788972760.019571:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445883b8 00000400:00000010:1.0:1788972760.019573:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972760.019578:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4a80 x1875872179112064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972776 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972760.019588:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4a80 x1875872179112064/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972776 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972760.019605:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459abc00. 02000000:00000010:1.0:1788972760.019613:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55000. 00000100:00000010:1.0:1788972760.019619:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb5500. 02000000:00000010:1.0:1788972760.019623:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab000. 02000000:00000010:1.0:1788972760.019625:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972760.019627:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4a80. 00000100:00000010:2.0:1788972760.713796:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4427ea20. 00000100:00000010:3.0:1788972760.714425:0:110986:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4427e4e0. 00000100:00000010:0.0:1788972765.135401:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353c380. 02000000:00000010:0.0:1788972765.135423:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb48503f00. 00000100:00000010:0.0:1788972765.135455:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb4353e680. 02000000:00000010:0.0:1788972765.135460:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb48503b00. 02000000:00000010:0.0:1788972765.135489:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459ab400. 00000400:00000200:0.0:1788972765.135497:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972765.135500:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163e18. 00000400:00000010:0.0:1788972765.135507:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084440. 00000100:00000200:0.0:1788972765.135514:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112320, portal 25 00000400:00000010:0.0:1788972765.135518:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084908. 00000100:00000200:0.0:1788972765.135522:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179112320, offset 0 00000400:00000200:0.0:1788972765.135531:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972765.135559:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972765.135566:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972765.135571:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972765.135575:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972765.135578:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972765.135581:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972765.135585:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [664:664:256:4294967295] 192.168.203.124@tcp LPNI seq info [664:664:8:4294967295] 00000400:00000200:0.0:1788972765.135589:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972765.135599:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972765.135605:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972765.135610:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb48503900. 00000800:00000200:0.0:1788972765.135617:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972765.135625:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972765.135629:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb48503900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972765.135644:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:0.0:1788972765.135649:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324540. 00000100:00000010:0.0:1788972765.135659:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb41324ba0. 02000000:00000010:0.0:1788972765.135679:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb459aa000. 00000400:00000200:0.0:1788972765.135683:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:0.0:1788972765.135686:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb48163738. 00000400:00000010:0.0:1788972765.135689:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48084ee0. 00000100:00000200:0.0:1788972765.135692:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112448, portal 10 00000400:00000010:0.0:1788972765.135697:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb480842a8. 00000100:00000200:0.0:1788972765.135699:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179112448, offset 0 00000400:00000200:0.0:1788972765.135703:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:0.0:1788972765.135707:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972765.135710:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:0.0:1788972765.135712:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972765.135715:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:0.0:1788972765.135717:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972765.135719:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:0.0:1788972765.135728:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [665:665:255:4294967295] 192.168.203.124@tcp LPNI seq info [665:665:7:4294967295] 00000400:00000200:0.0:1788972765.135732:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:0.0:1788972765.135738:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:0.0:1788972765.135751:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:0.0:1788972765.135755:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb48503100. 00000800:00000200:0.0:1788972765.135760:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:0.0:1788972765.135764:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:0.0:1788972765.135766:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb48503100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972765.135772:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972765.136025:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972765.136030:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb48503900. 00000400:00000200:2.0:1788972765.136035:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972765.136040:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972765.136045:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084908 00000400:00000010:2.0:1788972765.136047:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084908. 00000100:00000200:2.0:1788972765.136056:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353c380 x1875872179112320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972765.136129:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972765.136138:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb48503100. 00000400:00000200:2.0:1788972765.136141:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972765.136145:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972765.136149:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb480842a8 00000400:00000010:2.0:1788972765.136150:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb480842a8. 00000100:00000200:2.0:1788972765.136156:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb4353e680 x1875872179112448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972781 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972765.139222:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.139236:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972765.139247:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.139254:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.139266:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.139272:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14a00 00000400:00000200:3.0:1788972765.139279:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.139288:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x220d [1] + 224 00000400:00000200:3.0:1788972765.139345:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972765.139354:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972765.139375:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353e680 x1875872179112448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972781 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972765.140718:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163738. 00000400:00000200:0.0:1788972765.140730:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084ee0 00000400:00000010:0.0:1788972765.140733:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084ee0. 00000100:00000200:0.0:1788972765.140756:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353e680 x1875872179112448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972781 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972765.140768:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353e680 x1875872179112448/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972781 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972765.140802:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459aa000. 02000000:00000010:0.0:1788972765.140817:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb48503b00. 00000100:00000010:0.0:1788972765.140821:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353e680. 00000400:00000200:3.0:1788972765.141159:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.141179:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972765.141201:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.141207:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.141213:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.141218:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14980 00000400:00000200:3.0:1788972765.141223:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972765.141268:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2205 [1] + 224 00000400:00000200:3.0:1788972765.141284:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972765.141289:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972765.141340:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb4353c380 x1875872179112320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972781 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972765.141377:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb48163e18. 00000400:00000200:0.0:1788972765.141383:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48084440 00000400:00000010:0.0:1788972765.141385:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48084440. 00000100:00000200:0.0:1788972765.141394:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb4353c380 x1875872179112320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972781 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972765.141404:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb4353c380 x1875872179112320/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972781 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972765.141418:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb459ab400. 02000000:00000010:0.0:1788972765.141423:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb48503f00. 00000100:00000010:0.0:1788972765.141426:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb4353c380. 00000100:00000010:2.0:1788972765.833689:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324540. 00000100:00000010:3.0:1788972765.834606:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb41324ba0. 00000001:02000400:0.0:1788972768.637868:0:135064:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 614: lnet_legacy2yaml: interfaces enumeration with multiple IP ranges ========================================================== 12:52:47 (1788972767) 00000100:00000010:2.0:1788972770.250635:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788972770.250666:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e100. 00000100:00000010:2.0:1788972770.250713:0:470:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972770.250719:0:470:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb4604e400. 02000000:00000010:2.0:1788972770.250760:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124cc00. 00000400:00000200:2.0:1788972770.250773:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972770.250778:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788972770.250787:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972770.250796:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112704, portal 10 00000400:00000010:2.0:1788972770.250805:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972770.250809:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179112704, offset 0 00000400:00000200:2.0:1788972770.250821:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972770.250833:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.250844:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972770.250851:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972770.250856:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972770.250861:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972770.250865:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972770.250869:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [666:666:256:4294967295] 192.168.203.124@tcp LPNI seq info [666:666:8:4294967295] 00000400:00000200:2.0:1788972770.250874:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.250886:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972770.250894:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972770.250899:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604e600. 00000800:00000200:2.0:1788972770.250928:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972770.250938:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972770.250944:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604e600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972770.250965:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972770.250971:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11960. 00000100:00000010:2.0:1788972770.250983:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f110c0. 02000000:00000010:2.0:1788972770.251009:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb5124c400. 00000400:00000200:2.0:1788972770.251014:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972770.251017:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788972770.251022:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972770.251026:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112576, portal 25 00000400:00000010:2.0:1788972770.251030:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972770.251033:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179112576, offset 0 00000400:00000200:2.0:1788972770.251038:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972770.251042:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.251046:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972770.251049:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972770.251052:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972770.251054:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972770.251057:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972770.251060:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [667:667:255:4294967295] 192.168.203.124@tcp LPNI seq info [667:667:7:4294967295] 00000400:00000200:2.0:1788972770.251065:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.251084:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972770.251089:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972770.251094:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb4604ee00. 00000800:00000200:2.0:1788972770.251097:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972770.251102:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972770.251105:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb4604ee00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972770.251110:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972770.251178:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972770.251188:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604e600. 00000400:00000200:2.0:1788972770.251193:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972770.251199:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972770.251206:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972770.251208:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972770.251219:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179112704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972770.251990:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972770.251999:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb4604ee00. 00000400:00000200:2.0:1788972770.252003:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972770.252011:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972770.252017:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:2.0:1788972770.252019:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972770.252029:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179112576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972786 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972770.256408:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.256422:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972770.256433:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.256439:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.256446:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.256451:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14a80 00000400:00000200:2.0:1788972770.256457:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.256465:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2225 [1] + 224 00000400:00000200:2.0:1788972770.256497:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972770.256504:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972770.256524:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179112576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972786 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972770.256575:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:2.0:1788972770.256581:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972770.256584:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972770.256590:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179112576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972786 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972770.256608:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179112576/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972786 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972770.256641:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124c400. 02000000:00000010:2.0:1788972770.256648:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e100. 00000100:00000010:2.0:1788972770.256654:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 00000400:00000200:2.0:1788972770.263319:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.263332:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972770.263343:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.263349:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.263355:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.263360:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14b00 00000400:00000200:2.0:1788972770.263366:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972770.263373:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2215 [1] + 224 00000400:00000200:2.0:1788972770.263393:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972770.263399:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972770.263417:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179112704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972786 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972770.263463:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:2.0:1788972770.263470:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:2.0:1788972770.263473:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972770.263479:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179112704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972786 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972770.263489:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179112704/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972786 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972770.263532:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb5124cc00. 02000000:00000010:2.0:1788972770.263539:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb4604e400. 00000100:00000010:2.0:1788972770.263546:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 00000100:00000010:2.0:1788972770.951513:0:109433:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11960. 00000100:00000010:3.0:1788972770.952442:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f110c0. 00000100:00000010:2.0:1788972775.368562:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411500. 02000000:00000010:2.0:1788972775.368581:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285800. 00000100:00000010:2.0:1788972775.368614:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43411880. 02000000:00000010:2.0:1788972775.368619:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285300. 02000000:00000010:2.0:1788972775.368655:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0e00. 00000400:00000200:2.0:1788972775.368664:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972775.368669:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3840. 00000400:00000010:2.0:1788972775.368675:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0990. 00000100:00000200:2.0:1788972775.368683:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112832, portal 25 00000400:00000010:2.0:1788972775.368688:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972775.368690:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179112832, offset 0 00000400:00000200:2.0:1788972775.368699:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972775.368708:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.368716:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972775.368722:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972775.368726:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972775.368729:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972775.368732:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972775.368735:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [668:668:256:4294967295] 192.168.203.124@tcp LPNI seq info [668:668:8:4294967295] 00000400:00000200:2.0:1788972775.368740:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.368750:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972775.368756:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972775.368760:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285000. 00000800:00000200:2.0:1788972775.368767:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972775.368774:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972775.368778:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972775.368789:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972775.368794:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11a20. 00000100:00000010:2.0:1788972775.368804:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11c00. 02000000:00000010:2.0:1788972775.368822:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb482f0c00. 00000400:00000200:2.0:1788972775.368826:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972775.368836:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b30b0. 00000400:00000010:2.0:1788972775.368840:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0dd0. 00000100:00000200:2.0:1788972775.368843:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179112960, portal 10 00000400:00000010:2.0:1788972775.368848:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972775.368850:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179112960, offset 0 00000400:00000200:2.0:1788972775.368854:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972775.368859:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.368862:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972775.368865:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972775.368867:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972775.368869:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972775.368871:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972775.368874:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [669:669:255:4294967295] 192.168.203.124@tcp LPNI seq info [669:669:7:4294967295] 00000400:00000200:2.0:1788972775.368878:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.368883:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972775.368887:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972775.368891:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285400. 00000800:00000200:2.0:1788972775.368895:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972775.368899:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972775.368902:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972775.368907:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972775.368953:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972775.368956:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285000. 00000400:00000200:2.0:1788972775.368961:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972775.368965:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972775.368970:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972775.368973:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972775.368980:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411500 x1875872179112832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972775.369051:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972775.369054:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285400. 00000400:00000200:2.0:1788972775.369061:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972775.369065:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972775.369068:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972775.369070:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972775.369075:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43411880 x1875872179112960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972791 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972775.373628:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373640:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972775.373650:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373655:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373660:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373664:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14b80 00000400:00000200:2.0:1788972775.373670:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373676:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2235 [1] + 224 00000400:00000200:2.0:1788972775.373694:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972775.373698:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972775.373714:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411500 x1875872179112832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972791 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972775.373744:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373748:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:2.0:1788972775.373753:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373757:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373760:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373762:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14c00 00000400:00000200:2.0:1788972775.373766:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972775.373771:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2245 [1] + 224 00000400:00000200:2.0:1788972775.373781:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972775.373784:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:2.0:1788972775.373791:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43411880 x1875872179112960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972791 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972775.373957:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b30b0. 00000400:00000200:3.0:1788972775.373962:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0dd0 00000400:00000010:3.0:1788972775.373965:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0dd0. 00000100:00000200:3.0:1788972775.373972:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411880 x1875872179112960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972791 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972775.373981:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411880 x1875872179112960/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972791 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972775.374010:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3840. 00000400:00000200:3.0:1788972775.374013:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0990 00000400:00000010:3.0:1788972775.374015:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0990. 00000100:00000200:3.0:1788972775.374019:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43411500 x1875872179112832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972791 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972775.374027:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43411500 x1875872179112832/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972791 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972775.374041:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0c00. 02000000:00000010:3.0:1788972775.374046:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285300. 00000100:00000010:3.0:1788972775.374058:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411880. 02000000:00000010:3.0:1788972775.374062:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb482f0e00. 02000000:00000010:3.0:1788972775.374065:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285800. 00000100:00000010:3.0:1788972775.374067:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43411500. 00000100:00000010:3.0:1788972776.071355:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11c00. 00000100:00000010:2.0:1788972776.072641:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11a20. 00000001:02000400:1.0:1788972776.695771:0:135637:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 615: lnet_legacy2yaml: modprobe configuration file input ========================================================== 12:52:55 (1788972775) 00000100:00000010:2.0:1788972780.488040:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972780.488072:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285900. 00000100:00000010:2.0:1788972780.488118:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972780.488123:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285e00. 02000000:00000010:2.0:1788972780.488164:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50d53a00. 00000400:00000200:2.0:1788972780.488175:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972780.488180:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788972780.488188:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972780.488196:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113088, portal 25 00000400:00000010:2.0:1788972780.488201:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972780.488204:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179113088, offset 0 00000400:00000200:2.0:1788972780.488216:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972780.488225:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972780.488234:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972780.488240:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972780.488245:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972780.488249:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972780.488252:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972780.488256:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [670:670:256:4294967295] 192.168.203.124@tcp LPNI seq info [670:670:8:4294967295] 00000400:00000200:2.0:1788972780.488261:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972780.488271:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972780.488277:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972780.488282:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285c00. 00000800:00000200:2.0:1788972780.488289:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972780.488297:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972780.488327:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972780.488340:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972780.488346:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11720. 00000100:00000010:2.0:1788972780.488364:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11360. 02000000:00000010:2.0:1788972780.488385:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb50d52600. 00000400:00000200:2.0:1788972780.488389:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972780.488392:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3e70. 00000400:00000010:2.0:1788972780.488396:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972780.488398:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113216, portal 10 00000400:00000010:2.0:1788972780.488404:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972780.488405:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179113216, offset 0 00000400:00000200:2.0:1788972780.488410:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972780.488413:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972780.488416:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972780.488419:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972780.488422:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972780.488424:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972780.488426:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972780.488429:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [671:671:255:4294967295] 192.168.203.124@tcp LPNI seq info [671:671:7:4294967295] 00000400:00000200:2.0:1788972780.488433:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972780.488439:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972780.488443:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972780.488446:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb54285000. 00000800:00000200:2.0:1788972780.488450:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972780.488454:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972780.488457:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb54285000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972780.488462:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972780.488517:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972780.488520:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285c00. 00000400:00000200:2.0:1788972780.488526:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972780.488531:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972780.488537:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972780.488539:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972780.488548:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179113088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972780.488769:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972780.488772:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb54285000. 00000400:00000200:2.0:1788972780.488776:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972780.488780:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972780.488783:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:2.0:1788972780.488784:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972780.488790:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179113216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972796 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972780.495577:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495591:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972780.495601:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495607:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495613:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495617:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14c80 00000400:00000200:3.0:1788972780.495623:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495631:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2255 [1] + 224 00000400:00000200:3.0:1788972780.495645:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972780.495651:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972780.495669:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179113088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972796 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972780.495708:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495711:0:100235:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972780.495715:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495717:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495720:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495723:0:100235:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14d00 00000400:00000200:3.0:1788972780.495727:0:100235:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972780.495731:0:100235:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2265 [1] + 224 00000400:00000200:3.0:1788972780.495742:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972780.495746:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972780.495753:0:100235:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179113216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972796 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972780.496779:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3e70. 00000400:00000200:2.0:1788972780.496793:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:2.0:1788972780.496796:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972780.496810:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179113216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972796 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972780.496821:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179113216/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972796 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972780.496869:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788972780.496873:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972780.496876:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972780.496881:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179113088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972796 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972780.496889:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179113088/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972796 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972780.496907:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50d52600. 02000000:00000010:2.0:1788972780.496915:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285e00. 00000100:00000010:2.0:1788972780.496924:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972780.496929:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb50d53a00. 02000000:00000010:2.0:1788972780.496931:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285900. 00000100:00000010:2.0:1788972780.496934:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 00000100:00000010:3.0:1788972781.191403:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11360. 00000100:00000010:2.0:1788972781.191520:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb50f11720. 00000001:02000400:3.0:1788972784.866233:0:136211:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 616: lnet_legacy2yaml: combining modprobe file with explicit ip2nets file ========================================================== 12:53:03 (1788972783) 00000100:00000010:1.0:1788972785.608467:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb4a80. 02000000:00000010:1.0:1788972785.608500:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972785.608540:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb42fb5500. 02000000:00000010:1.0:1788972785.608546:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb51f55000. 02000000:00000010:1.0:1788972785.608591:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2400. 00000400:00000200:1.0:1788972785.608603:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972785.608608:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1c08. 00000400:00000010:1.0:1788972785.608617:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972785.608626:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113344, portal 25 00000400:00000010:1.0:1788972785.608633:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb44588198. 00000100:00000200:1.0:1788972785.608638:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179113344, offset 0 00000400:00000200:1.0:1788972785.608649:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972785.608659:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972785.608668:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972785.608674:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972785.608689:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972785.608693:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972785.608696:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972785.608700:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [672:672:256:4294967295] 192.168.203.124@tcp LPNI seq info [672:672:8:4294967295] 00000400:00000200:1.0:1788972785.608705:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972785.608717:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972785.608724:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972785.608730:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55300. 00000800:00000200:1.0:1788972785.608737:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972785.608747:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972785.608874:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972785.608898:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:1.0:1788972785.608904:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4427e360. 00000100:00000010:1.0:1788972785.608917:0:100241:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb4427e000. 02000000:00000010:1.0:1788972785.608941:0:100241:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb513f2e00. 00000400:00000200:1.0:1788972785.608946:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:1.0:1788972785.608949:0:100241:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb451f1cb8. 00000400:00000010:1.0:1788972785.608953:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788972785.608956:0:100241:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113472, portal 10 00000400:00000010:1.0:1788972785.608962:0:100241:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb445882a8. 00000100:00000200:1.0:1788972785.608964:0:100241:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179113472, offset 0 00000400:00000200:1.0:1788972785.608971:0:100241:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:1.0:1788972785.608975:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972785.609212:0:100234:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972785.609218:0:100234:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55300. 00000400:00000200:3.0:1788972785.609224:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972785.609229:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972785.609236:0:100234:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb44588198 00000400:00000010:3.0:1788972785.609239:0:100234:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb44588198. 00000100:00000200:3.0:1788972785.609251:0:100234:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb4a80 x1875872179113344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972785.609664:0:100241:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:1.0:1788972785.609670:0:100241:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972785.609674:0:100241:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:1.0:1788972785.609677:0:100241:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972785.609680:0:100241:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:1.0:1788972785.609683:0:100241:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [673:673:256:4294967295] 192.168.203.124@tcp LPNI seq info [673:673:8:4294967295] 00000400:00000200:1.0:1788972785.609688:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:1.0:1788972785.609697:0:100241:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:1.0:1788972785.609714:0:100241:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:1.0:1788972785.609718:0:100241:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb51f55900. 00000800:00000200:1.0:1788972785.609725:0:100241:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:1.0:1788972785.609731:0:100241:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:1.0:1788972785.609735:0:100241:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb51f55900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972785.609746:0:100241:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:3.0:1788972785.610143:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972785.610147:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb51f55900. 00000400:00000200:3.0:1788972785.610153:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972785.610159:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:3.0:1788972785.610165:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445882a8 00000400:00000010:3.0:1788972785.610167:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445882a8. 00000100:00000200:3.0:1788972785.610176:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb42fb5500 x1875872179113472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972801 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972785.618023:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618037:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972785.618056:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618062:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618069:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618074:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14d80 00000400:00000200:3.0:1788972785.618080:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618088:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2275 [1] + 224 00000400:00000200:3.0:1788972785.618101:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972785.618106:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972785.618127:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb4a80 x1875872179113344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972801 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972785.618169:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618188:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972785.618193:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618196:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618199:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618202:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14e00 00000400:00000200:3.0:1788972785.618206:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972785.618210:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x227d [1] + 224 00000400:00000200:3.0:1788972785.618220:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972785.618224:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972785.618232:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb42fb5500 x1875872179113472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972801 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972785.618641:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1cb8. 00000400:00000200:1.0:1788972785.618648:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445887f8 00000400:00000010:1.0:1788972785.618651:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445887f8. 00000100:00000200:1.0:1788972785.618658:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb5500 x1875872179113472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972801 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972785.618667:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb5500 x1875872179113472/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972801 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972785.618702:0:100241:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb451f1c08. 00000400:00000200:1.0:1788972785.618705:0:100241:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb445883b8 00000400:00000010:1.0:1788972785.618707:0:100241:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb445883b8. 00000100:00000200:1.0:1788972785.618710:0:100241:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb42fb4a80 x1875872179113344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972801 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972785.618717:0:100241:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb42fb4a80 x1875872179113344/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972801 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972785.618732:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2e00. 02000000:00000010:1.0:1788972785.618740:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55000. 00000100:00000010:1.0:1788972785.618745:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb5500. 02000000:00000010:1.0:1788972785.618761:0:100241:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb513f2400. 02000000:00000010:1.0:1788972785.618763:0:100241:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb51f55500. 00000100:00000010:1.0:1788972785.618765:0:100241:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb42fb4a80. 00000100:00000010:3.0:1788972786.314255:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4427e000. 00000100:00000010:2.0:1788972786.315785:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb4427e360. 00000100:00000010:3.0:1788972790.728702:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c5180. 02000000:00000010:3.0:1788972790.728721:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1500. 00000100:00000010:3.0:1788972790.728750:0:114270:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb434c7480. 02000000:00000010:3.0:1788972790.728755:0:114270:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb42ef1900. 02000000:00000010:3.0:1788972790.728788:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51150600. 00000400:00000200:3.0:1788972790.728799:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972790.728803:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78580. 00000400:00000010:3.0:1788972790.728809:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081c38. 00000100:00000200:3.0:1788972790.728817:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113600, portal 25 00000400:00000010:3.0:1788972790.728822:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081440. 00000100:00000200:3.0:1788972790.728825:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179113600, offset 0 00000400:00000200:3.0:1788972790.728833:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972790.728841:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.728849:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972790.728853:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972790.728857:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972790.728861:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972790.728864:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972790.728867:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [674:674:256:4294967295] 192.168.203.124@tcp LPNI seq info [674:674:8:4294967295] 00000400:00000200:3.0:1788972790.728881:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.728891:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972790.728897:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972790.728901:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1100. 00000800:00000200:3.0:1788972790.728908:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972790.728915:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972790.728920:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972790.728979:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:3.0:1788972790.728984:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9420. 00000100:00000010:3.0:1788972790.728996:0:100243:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb51fd9660. 02000000:00000010:3.0:1788972790.729027:0:100243:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb51151600. 00000400:00000200:3.0:1788972790.729032:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:3.0:1788972790.729034:0:100243:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb43e78d68. 00000400:00000010:3.0:1788972790.729038:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081bb0. 00000100:00000200:3.0:1788972790.729041:0:100243:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113728, portal 10 00000400:00000010:3.0:1788972790.729046:0:100243:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb48081aa0. 00000100:00000200:3.0:1788972790.729048:0:100243:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179113728, offset 0 00000400:00000200:3.0:1788972790.729052:0:100243:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:3.0:1788972790.729055:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.730069:0:100243:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:3.0:1788972790.730076:0:100243:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972790.730080:0:100243:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:3.0:1788972790.730084:0:100243:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972790.730088:0:100243:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:3.0:1788972790.730091:0:100243:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [675:675:255:4294967295] 192.168.203.124@tcp LPNI seq info [675:675:7:4294967295] 00000400:00000200:3.0:1788972790.730097:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.730107:0:100243:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:3.0:1788972790.730113:0:100243:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:3.0:1788972790.730118:0:100243:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb42ef1f00. 00000800:00000200:3.0:1788972790.730124:0:100243:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:3.0:1788972790.730136:0:100243:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:3.0:1788972790.730141:0:100243:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb42ef1f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972790.730149:0:100243:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972790.731443:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972790.731453:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1100. 00000400:00000200:2.0:1788972790.731460:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972790.731468:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972790.731475:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081440 00000400:00000010:2.0:1788972790.731478:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081440. 00000100:00000200:2.0:1788972790.731489:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c5180 x1875872179113600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972790.731564:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972790.731566:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb42ef1f00. 00000400:00000200:2.0:1788972790.731570:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972790.731573:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972790.731576:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081aa0 00000400:00000010:2.0:1788972790.731578:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081aa0. 00000100:00000200:2.0:1788972790.731584:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb434c7480 x1875872179113728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972806 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972790.739861:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.739872:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972790.739881:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.739886:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.739890:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.739894:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e14f00 00000400:00000200:3.0:1788972790.739899:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.739905:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x2295 [1] + 224 00000400:00000200:3.0:1788972790.739916:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972790.739920:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972790.739933:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c7480 x1875872179113728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972806 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972790.740004:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.740007:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972790.740020:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.740022:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.740025:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.740028:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14e80 00000400:00000200:3.0:1788972790.740031:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972790.740035:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x2285 [1] + 224 00000400:00000200:3.0:1788972790.740042:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972790.740045:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972790.740052:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb434c5180 x1875872179113600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972806 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972790.744322:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78d68. 00000400:00000200:2.0:1788972790.744338:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081bb0 00000400:00000010:2.0:1788972790.744340:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081bb0. 00000100:00000200:2.0:1788972790.744357:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c7480 x1875872179113728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972806 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972790.744370:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c7480 x1875872179113728/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972806 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972790.744408:0:100243:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb43e78580. 00000400:00000200:2.0:1788972790.744411:0:100243:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb48081c38 00000400:00000010:2.0:1788972790.744413:0:100243:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb48081c38. 00000100:00000200:2.0:1788972790.744418:0:100243:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb434c5180 x1875872179113600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972806 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972790.744425:0:100243:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb434c5180 x1875872179113600/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972806 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972790.744469:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51151600. 02000000:00000010:2.0:1788972790.744476:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1900. 00000100:00000010:2.0:1788972790.744480:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c7480. 02000000:00000010:2.0:1788972790.744484:0:100243:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb51150600. 02000000:00000010:2.0:1788972790.744487:0:100243:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb42ef1500. 00000100:00000010:2.0:1788972790.744490:0:100243:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb434c5180. 00000100:00000010:2.0:1788972791.431464:0:129626:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9420. 00000100:00000010:3.0:1788972791.431670:0:119801:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff98bb51fd9660. 00000001:02000400:0.0:1788972792.853752:0:136785:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 617: lnet_legacy2yaml: round-trip tunables from LNet to modprobe to YAML ========================================================== 12:53:11 (1788972791) 00000001:02000400:1.0:1788972792.979052:0:136792:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000001:02000400:2.0:1788972794.642543:0:136925:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_617: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 00000100:00000010:2.0:1788972795.851082:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972795.851112:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285a00. 00000100:00000010:2.0:1788972795.851149:0:117224:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff98bb43410000. 02000000:00000010:2.0:1788972795.851157:0:117224:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff98bb54285e00. 02000000:00000010:2.0:1788972795.851191:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a600. 00000400:00000200:2.0:1788972795.851202:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972795.851206:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3948. 00000400:00000010:2.0:1788972795.851213:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972795.851221:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113984, portal 10 00000400:00000010:2.0:1788972795.851228:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972795.851231:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872179113984, offset 0 00000400:00000200:2.0:1788972795.851240:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972795.851249:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972795.851267:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972795.851272:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972795.851276:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972795.851279:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972795.851282:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972795.851285:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [676:676:256:4294967295] 192.168.203.124@tcp LPNI seq info [676:676:8:4294967295] 00000400:00000200:2.0:1788972795.851289:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972795.851315:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972795.851332:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972795.851351:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb496ee000. 00000800:00000200:2.0:1788972795.851358:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972795.851366:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972795.851370:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb496ee000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972795.851384:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000100:00000010:2.0:1788972795.851389:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11d20. 00000100:00000010:2.0:1788972795.851398:0:100242:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff98bb50f11f60. 02000000:00000010:2.0:1788972795.851424:0:100242:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff98bb4166a000. 00000400:00000200:2.0:1788972795.851429:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000010:2.0:1788972795.851431:0:100242:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff98bb453b3e70. 00000400:00000010:2.0:1788972795.851435:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972795.851437:0:100242:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872179113856, portal 25 00000400:00000010:2.0:1788972795.851441:0:100242:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972795.851442:0:100242:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872179113856, offset 0 00000400:00000200:2.0:1788972795.851446:0:100242:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.124@tcp 00000400:00000200:2.0:1788972795.851450:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972795.851453:0:100242:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.124@tcp local destination 00000400:00000200:2.0:1788972795.851456:0:100242:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972795.851458:0:100242:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.24@tcp 00000400:00000200:2.0:1788972795.851460:0:100242:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.24@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972795.851462:0:100242:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.124@tcp 00000400:00000200:2.0:1788972795.851465:0:100242:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.24@tcp NI seq info: [677:677:255:4294967295] 192.168.203.124@tcp LPNI seq info [677:677:7:4294967295] 00000400:00000200:2.0:1788972795.851469:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:2.0:1788972795.851475:0:100242:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp:) -> 192.168.203.124@tcp(192.168.203.124@tcp:192.168.203.124@tcp) : PUT try# 0 00000800:00000200:2.0:1788972795.851480:0:100242:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.124@tcp 00000800:00000010:2.0:1788972795.851484:0:100242:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff98bb53a9cb00. 00000800:00000200:2.0:1788972795.851487:0:100242:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff98bb434f4cc0] -> 12345-192.168.203.124@tcp (5) 00000800:00000200:2.0:1788972795.851491:0:100242:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.124@tcp ip 192.168.203.124:988 00000800:00000200:2.0:1788972795.851494:0:100242:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff98bb53a9cb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972795.851497:0:100242:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000800:00000200:2.0:1788972795.851549:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972795.851553:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb496ee000. 00000400:00000200:2.0:1788972795.851559:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972795.851565:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972795.851570:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0d48 00000400:00000010:2.0:1788972795.851573:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0d48. 00000100:00000200:2.0:1788972795.851581:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43410000 x1875872179113984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972795.851672:0:100235:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972795.851675:0:100235:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff98bb53a9cb00. 00000400:00000200:2.0:1788972795.851678:0:100235:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972795.851682:0:100235:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000400:00000200:2.0:1788972795.851684:0:100235:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0220 00000400:00000010:2.0:1788972795.851686:0:100235:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0220. 00000100:00000200:2.0:1788972795.851690:0:100235:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff98bb43413480 x1875872179113856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972811 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972795.859918:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.859935:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972795.859944:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.859949:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.859953:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.859957:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 25 MB=0x6aa1886e14f80 00000400:00000200:3.0:1788972795.859963:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.859968:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.124@tcp of length 224/224 into md 0x22b5 [1] + 224 00000400:00000200:3.0:1788972795.859990:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972795.859996:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972795.860011:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43413480 x1875872179113856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972811 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972795.860037:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.860041:0:100234:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.24@tcp(192.168.203.24@tcp) <- 192.168.203.124@tcp : PUT - for me 00000400:00000200:3.0:1788972795.860045:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.860047:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.860049:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.860052:0:100234:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.124@tcp of length 224 into portal 10 MB=0x6aa1886e15000 00000400:00000200:3.0:1788972795.860055:0:100234:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.124@tcp to cpt 1 00000400:00000200:3.0:1788972795.860059:0:100234:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.124@tcp of length 224/224 into md 0x22a5 [1] + 224 00000400:00000200:3.0:1788972795.860069:0:100234:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972795.860072:0:100234:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.24@tcp->192.168.203.124@tcp: PUT: OK 00000100:00000200:3.0:1788972795.860078:0:100234:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff98bb43410000 x1875872179113984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972811 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972795.860142:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3e70. 00000400:00000200:2.0:1788972795.860147:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0440 00000400:00000010:2.0:1788972795.860149:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0440. 00000100:00000200:2.0:1788972795.860155:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43413480 x1875872179113856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972811 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972795.860163:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43413480 x1875872179113856/t0(0) o400->MGC192.168.203.124@tcp@192.168.203.124@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972811 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972795.860184:0:100242:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff98bb453b3948. 00000400:00000200:2.0:1788972795.860186:0:100242:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff98bb447b0f68 00000400:00000010:2.0:1788972795.860188:0:100242:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff98bb447b0f68. 00000100:00000200:2.0:1788972795.860191:0:100242:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff98bb43410000 x1875872179113984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972811 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972795.860198:0:100242:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff98bb43410000 x1875872179113984/t0(0) o400->lustre-MDT0000-mdc-ffff98bb44a52000@192.168.203.124@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972811 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972795.860217:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a000. 02000000:00000010:2.0:1788972795.860229:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285a00. 00000100:00000010:2.0:1788972795.860234:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43413480. 02000000:00000010:2.0:1788972795.860238:0:100242:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff98bb4166a600. 02000000:00000010:2.0:1788972795.860240:0:100242:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff98bb54285e00. 00000100:00000010:2.0:1788972795.860242:0:100242:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff98bb43410000. Debug log: 3306 lines, 3306 kept, 0 dropped, 0 bad.