00000100:00000010:2.0F:1788972449.046412:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e300.
02000000:00000010:2.0:1788972449.046437:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c500.
00000100:00000010:2.0:1788972449.046468:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80.
02000000:00000010:2.0:1788972449.046486:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4cd00.
00000100:00000010:2.0:1788972449.046493:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000.
02000000:00000010:2.0:1788972449.046496:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c200.
02000000:00000010:2.0:1788972449.046527:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ea00.
00000400:00000200:2.0:1788972449.046536:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972449.046540:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01261b8.
00000400:00000010:2.0:1788972449.046546:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8.
00000100:00000200:2.0:1788972449.046553:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189611648, portal 25
00000400:00000010:2.0:1788972449.046558:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28.
00000100:00000200:2.0:1788972449.046561:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189611648, offset 0
00000400:00000200:2.0:1788972449.046569:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972449.046577:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972449.046584:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788972449.046589:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972449.046593:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788972449.046596:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972449.046599:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972449.046603:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [770:770:256:4294967295] 192.168.203.150@tcp LPNI seq info [770:770:8:4294967295]
00000400:00000200:2.0:1788972449.046607:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972449.046616:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972449.046621:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972449.046626:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c600.
00000800:00000200:2.0:1788972449.046632:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972449.046639:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972449.046643:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972449.046657:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788972449.046663:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e2a0.
00000100:00000010:2.0:1788972449.046673:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed80.
02000000:00000010:2.0:1788972449.046683:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fc00.
00000400:00000200:2.0:1788972449.046692:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972449.046695:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126ec8.
00000400:00000010:2.0:1788972449.046698:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0.
00000100:00000200:2.0:1788972449.046701:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189611904, portal 10
00000400:00000010:2.0:1788972449.046706:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788972449.046709:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189611904, offset 0
00000400:00000200:2.0:1788972449.046713:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972449.046717:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972449.046720:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788972449.046723:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972449.046725:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788972449.046727:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972449.046729:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972449.046732:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [771:771:255:4294967295] 192.168.203.150@tcp LPNI seq info [771:771:7:4294967295]
00000400:00000200:2.0:1788972449.046736:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972449.046741:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972449.046745:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972449.046748:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c100.
00000800:00000200:2.0:1788972449.046752:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972449.046756:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972449.046758:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972449.046763:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788972449.046781:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ec00.
00000400:00000200:2.0:1788972449.046784:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972449.046786:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01267e8.
00000400:00000010:2.0:1788972449.046789:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b990.
00000100:00000200:2.0:1788972449.046791:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189611776, portal 10
00000400:00000010:2.0:1788972449.046795:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8.
00000100:00000200:2.0:1788972449.046796:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189611776, offset 0
00000400:00000200:2.0:1788972449.046800:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972449.046803:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972449.046837:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788972449.046840:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972449.046843:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788972449.046845:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972449.046847:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972449.046850:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [772:772:254:4294967295] 192.168.203.150@tcp LPNI seq info [772:772:6:4294967295]
00000400:00000200:2.0:1788972449.046853:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972449.046858:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972449.046863:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972449.046867:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4cb00.
00000800:00000200:2.0:1788972449.046871:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972449.046875:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972449.046878:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4cb00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972449.046883:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788972449.046928:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972449.046931:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c600.
00000400:00000200:2.0:1788972449.046937:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972449.046942:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788972449.046946:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28
00000400:00000010:2.0:1788972449.046949:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28.
00000100:00000200:2.0:1788972449.046958:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840e300 x1875872189611648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788972449.046979:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972449.046981:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c100.
00000400:00000200:2.0:1788972449.046984:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972449.046987:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788972449.046989:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8
00000400:00000010:2.0:1788972449.046991:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788972449.046996:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840c000 x1875872189611904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788972449.047223:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972449.047231:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4cb00.
00000400:00000200:2.0:1788972449.047234:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972449.047238:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788972449.047242:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8
00000400:00000010:2.0:1788972449.047244:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8.
00000100:00000200:2.0:1788972449.047249:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ca80 x1875872189611776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0F:1788972449.060096:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060113:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972449.060121:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060126:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060132:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060136:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817f80
00000400:00000200:3.0:1788972449.060141:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060148:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x25c5 [1] + 224
00000400:00000200:3.0:1788972449.060161:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972449.060165:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972449.060181:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840c000 x1875872189611904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972449.060210:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060213:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972449.060217:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060219:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060222:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060224:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817f00
00000400:00000200:3.0:1788972449.060227:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060230:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x25d5 [1] + 224
00000400:00000200:3.0:1788972449.060236:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972449.060239:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972449.060244:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ca80 x1875872189611776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972449.060257:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060261:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972449.060264:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060267:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060269:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060271:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887817e80
00000400:00000200:3.0:1788972449.060274:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972449.060284:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x25b5 [1] + 224
00000400:00000200:3.0:1788972449.060293:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972449.060296:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972449.060303:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840e300 x1875872189611648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972449.060330:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01267e8.
00000400:00000200:3.0:1788972449.060334:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b990
00000400:00000010:3.0:1788972449.060336:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b990.
00000100:00000200:3.0:1788972449.060341:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ca80 x1875872189611776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972449.060350:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ca80 x1875872189611776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972449.060376:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01261b8.
00000400:00000200:3.0:1788972449.060379:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8
00000400:00000010:3.0:1788972449.060381:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8.
00000100:00000200:3.0:1788972449.060385:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840e300 x1875872189611648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972449.060393:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840e300 x1875872189611648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972449.060403:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126ec8.
00000400:00000200:3.0:1788972449.060405:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0
00000400:00000010:3.0:1788972449.060407:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0.
00000100:00000200:3.0:1788972449.060410:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840c000 x1875872189611904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972449.060416:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840c000 x1875872189611904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972465 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972449.060429:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ec00.
02000000:00000010:3.0:1788972449.060433:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4cd00.
00000100:00000010:3.0:1788972449.060438:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80.
02000000:00000010:3.0:1788972449.060447:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ea00.
02000000:00000010:3.0:1788972449.060449:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c500.
00000100:00000010:3.0:1788972449.060451:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e300.
02000000:00000010:3.0:1788972449.060454:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fc00.
02000000:00000010:3.0:1788972449.060456:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c200.
00000100:00000010:3.0:1788972449.060458:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000.
00000100:00000010:3.0:1788972450.065907:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed80.
00000100:00000010:2.0:1788972450.065977:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e2a0.
00000001:02000400:3.0:1788972452.903658:0:122213:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_410 skipping ALWAYS excluded test 410
00000100:00000010:2.0:1788972454.163433:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ea00.
02000000:00000010:2.0:1788972454.163465:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c600.
00000100:00000010:2.0:1788972454.163500:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788972454.163506:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c100.
00000100:00000010:2.0:1788972454.163512:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100.
02000000:00000010:2.0:1788972454.163516:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c200.
02000000:00000010:3.0:1788972454.164715:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8002400.
00000400:00000200:3.0:1788972454.164729:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788972454.164734:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68.
00000400:00000010:3.0:1788972454.164740:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873a18.
00000100:00000200:3.0:1788972454.164747:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189612032, portal 25
00000400:00000010:3.0:1788972454.164752:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0.
00000100:00000200:3.0:1788972454.164755:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189612032, offset 0
00000400:00000200:3.0:1788972454.164764:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972454.164772:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972454.164780:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972454.164785:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972454.164790:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972454.164793:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972454.164796:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972454.164800:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [773:773:256:4294967295] 192.168.203.150@tcp LPNI seq info [773:773:8:4294967295]
00000400:00000200:3.0:1788972454.164804:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972454.164833:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972454.164840:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788972454.164845:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d200.
00000800:00000200:3.0:1788972454.164851:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788972454.164858:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788972454.164862:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972454.164877:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788972454.164892:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d2a0.
00000100:00000010:3.0:1788972454.164901:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c654d8a0.
02000000:00000010:3.0:1788972454.164913:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8003400.
00000400:00000200:3.0:1788972454.164917:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788972454.164920:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210.
00000400:00000010:3.0:1788972454.164923:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873d48.
00000100:00000200:3.0:1788972454.164926:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189612288, portal 10
00000400:00000010:3.0:1788972454.164932:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788972454.164934:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189612288, offset 0
00000400:00000200:3.0:1788972454.164938:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972454.164942:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972454.164946:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972454.164949:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972454.164951:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972454.164954:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972454.164956:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972454.164959:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [774:774:255:4294967295] 192.168.203.150@tcp LPNI seq info [774:774:7:4294967295]
00000400:00000200:3.0:1788972454.164963:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972454.164968:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972454.164972:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788972454.164976:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d600.
00000800:00000200:3.0:1788972454.164979:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788972454.164983:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788972454.164986:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972454.166507:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788972454.167058:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0fb7a00.
00000400:00000200:3.0:1788972454.167067:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788972454.167072:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688.
00000400:00000010:3.0:1788972454.167079:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788972454.167085:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189612160, portal 10
00000400:00000010:3.0:1788972454.167091:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788972454.167102:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189612160, offset 0
00000400:00000200:3.0:1788972454.167110:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972454.167117:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972454.167124:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972454.167128:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972454.167131:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972454.167134:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972454.167137:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972454.167140:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [775:775:254:4294967295] 192.168.203.150@tcp LPNI seq info [775:775:6:4294967295]
00000400:00000200:3.0:1788972454.167144:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972454.167153:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972454.168096:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788972454.168103:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518de00.
00000800:00000200:3.0:1788972454.168110:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788972454.168117:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788972454.168122:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518de00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972454.168132:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788972454.169119:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972454.169126:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d200.
00000400:00000200:3.0:1788972454.169131:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972454.169138:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972454.169144:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0
00000400:00000010:3.0:1788972454.169147:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0.
00000100:00000200:3.0:1788972454.170114:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ea00 x1875872189612032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972470 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972454.170154:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972454.171099:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d600.
00000400:00000200:3.0:1788972454.171105:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972454.171112:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972454.171118:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8
00000400:00000010:3.0:1788972454.171120:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788972454.171141:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f100 x1875872189612288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972454.173452:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972454.173462:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518de00.
00000400:00000200:3.0:1788972454.173468:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972454.173477:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972454.173484:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873660
00000400:00000010:3.0:1788972454.173487:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873660.
00000100:00000200:3.0:1788972454.173497:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ed80 x1875872189612160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788972454.190114:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190125:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788972454.190136:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190141:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190147:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190152:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818100
00000400:00000200:2.0:1788972454.190410:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190418:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x25f5 [1] + 224
00000400:00000200:2.0:1788972454.190431:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972454.190447:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788972454.190461:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f100 x1875872189612288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788972454.190503:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190507:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788972454.190512:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190514:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190528:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190531:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818080
00000400:00000200:2.0:1788972454.190535:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190538:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2605 [1] + 224
00000400:00000200:2.0:1788972454.190546:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972454.190549:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788972454.190556:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ed80 x1875872189612160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788972454.190570:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190573:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788972454.190578:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190580:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190583:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190585:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887818000
00000400:00000200:2.0:1788972454.190589:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972454.190593:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x25e5 [1] + 224
00000400:00000200:2.0:1788972454.190602:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972454.190605:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788972454.190611:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ea00 x1875872189612032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972454.190641:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688.
00000400:00000200:2.0:1788972454.190646:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908
00000400:00000010:2.0:1788972454.190648:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908.
00000100:00000200:2.0:1788972454.190654:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ed80 x1875872189612160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972454.190662:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ed80 x1875872189612160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972454.190688:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68.
00000400:00000200:2.0:1788972454.190691:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873a18
00000400:00000010:2.0:1788972454.190699:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873a18.
00000100:00000200:2.0:1788972454.190705:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ea00 x1875872189612032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972454.190713:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ea00 x1875872189612032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788972454.190726:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210.
00000400:00000200:2.0:1788972454.190728:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873d48
00000400:00000010:2.0:1788972454.190730:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873d48.
00000100:00000200:2.0:1788972454.190734:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f100 x1875872189612288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972454.190741:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f100 x1875872189612288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972470 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972454.190756:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0fb7a00.
02000000:00000010:2.0:1788972454.190764:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c100.
00000100:00000010:2.0:1788972454.190770:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80.
02000000:00000010:2.0:1788972454.190774:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8002400.
02000000:00000010:2.0:1788972454.190777:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c600.
00000100:00000010:2.0:1788972454.190779:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ea00.
02000000:00000010:2.0:1788972454.190782:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8003400.
02000000:00000010:2.0:1788972454.190784:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c200.
00000100:00000010:2.0:1788972454.190786:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100.
00000001:02000400:2.0:1788972454.998547:0:122366:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 450: Check import of multiple interfaces ========================================================== 12:47:33 (1788972453)
00000100:00000010:3.0:1788972455.187871:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d8a0.
00000100:00000010:2.0:1788972455.191720:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c654d2a0.
00000001:02000400:1.0F:1788972455.255860:0:122377:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:3.0:1788972456.922738:0:122508:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_450: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
Debug log: 318 lines, 318 kept, 0 dropped, 0 bad.