00000100:00000010:3.0F:1788972397.845171:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793480.
02000000:00000010:3.0:1788972397.845204:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d800.
00000100:00000010:3.0:1788972397.847554:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100.
02000000:00000010:3.0:1788972397.847578:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d200.
00000100:00000010:3.0:1788972397.847596:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000.
02000000:00000010:3.0:1788972397.847601:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d600.
02000000:00000010:3.0:1788972397.847646:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3f3e600.
00000400:00000200:3.0:1788972397.847656:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788972397.847661:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436210.
00000400:00000010:3.0:1788972397.847667:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788972397.847675:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189607552, portal 25
00000400:00000010:3.0:1788972397.847680:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788972397.847683:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189607552, offset 0
00000400:00000200:3.0:1788972397.847694:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972397.847704:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.847711:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972397.847717:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972397.847721:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972397.847724:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972397.847727:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972397.847731:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [738:738:256:4294967295] 192.168.203.150@tcp LPNI seq info [738:738:8:4294967295]
00000400:00000200:3.0:1788972397.847736:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.847746:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972397.847752:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788972397.847757:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518de00.
00000800:00000200:3.0:1788972397.847764:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788972397.847772:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788972397.847777:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518de00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972397.847795:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:3.0:1788972397.847801:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09a80.
00000100:00000010:3.0:1788972397.847828:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09ae0.
02000000:00000010:3.0:1788972397.847851:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3f3f200.
00000400:00000200:3.0:1788972397.847870:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788972397.847873:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10.
00000400:00000010:3.0:1788972397.847877:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873bb0.
00000100:00000200:3.0:1788972397.847879:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189607808, portal 10
00000400:00000010:3.0:1788972397.847886:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788972397.847888:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189607808, offset 0
00000400:00000200:3.0:1788972397.847892:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972397.847896:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.847900:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972397.847903:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972397.847905:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972397.847907:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972397.847910:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972397.847912:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [739:739:255:4294967295] 192.168.203.150@tcp LPNI seq info [739:739:7:4294967295]
00000400:00000200:3.0:1788972397.847916:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.847922:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972397.847927:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788972397.847930:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d300.
00000800:00000200:3.0:1788972397.847934:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788972397.847938:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788972397.847941:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972397.847947:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:3.0:1788972397.847969:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c3f3e200.
00000400:00000200:3.0:1788972397.847973:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:3.0:1788972397.847975:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d68.
00000400:00000010:3.0:1788972397.847979:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873c38.
00000100:00000200:3.0:1788972397.847981:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189607680, portal 10
00000400:00000010:3.0:1788972397.847986:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873dd0.
00000100:00000200:3.0:1788972397.847987:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189607680, offset 0
00000400:00000200:3.0:1788972397.847991:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972397.847995:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.848004:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972397.848007:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1788972397.848010:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972397.848012:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972397.848014:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972397.848016:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [740:740:254:4294967295] 192.168.203.150@tcp LPNI seq info [740:740:6:4294967295]
00000400:00000200:3.0:1788972397.848020:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.848025:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972397.848029:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:3.0:1788972397.848032:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518da00.
00000800:00000200:3.0:1788972397.848036:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:3.0:1788972397.848040:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:3.0:1788972397.848042:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518da00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972397.848048:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788972397.848132:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972397.848137:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518de00.
00000400:00000200:3.0:1788972397.848143:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972397.848147:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972397.848152:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873908
00000400:00000010:3.0:1788972397.848154:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873908.
00000100:00000200:3.0:1788972397.848163:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793480 x1875872189607552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972413 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972397.848183:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972397.848185:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d300.
00000400:00000200:3.0:1788972397.848187:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972397.848190:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972397.848192:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48737f8
00000400:00000010:3.0:1788972397.848194:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48737f8.
00000100:00000200:3.0:1788972397.848198:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8790000 x1875872189607808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972397.848260:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972397.848267:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518da00.
00000400:00000200:3.0:1788972397.848270:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972397.848274:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972397.848277:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873dd0
00000400:00000010:3.0:1788972397.848279:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873dd0.
00000100:00000200:3.0:1788972397.848285:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793100 x1875872189607680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972397.856158:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856169:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972397.856179:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856184:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856188:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856192:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887816f80
00000400:00000200:3.0:1788972397.856198:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856204:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2435 [1] + 224
00000400:00000200:3.0:1788972397.856216:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972397.856221:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972397.856236:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8790000 x1875872189607808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972397.856263:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856267:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972397.856271:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856274:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856277:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856279:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887816f00
00000400:00000200:3.0:1788972397.856282:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856286:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2445 [1] + 224
00000400:00000200:3.0:1788972397.856292:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972397.856296:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972397.856302:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793100 x1875872189607680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972397.856347:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856350:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972397.856354:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856356:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856359:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856361:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887816e80
00000400:00000200:3.0:1788972397.856364:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972397.856368:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2425 [1] + 224
00000400:00000200:3.0:1788972397.856376:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972397.856380:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972397.856401:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793480 x1875872189607552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972397.856431:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d68.
00000400:00000200:3.0:1788972397.856436:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873c38
00000400:00000010:3.0:1788972397.856438:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873c38.
00000100:00000200:3.0:1788972397.856443:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793100 x1875872189607680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972397.856452:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793100 x1875872189607680/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972397.856482:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436210.
00000400:00000200:3.0:1788972397.856485:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8
00000400:00000010:3.0:1788972397.856487:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8.
00000100:00000200:3.0:1788972397.856491:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793480 x1875872189607552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972397.856499:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793480 x1875872189607552/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972397.856509:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10.
00000400:00000200:3.0:1788972397.856512:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873bb0
00000400:00000010:3.0:1788972397.856519:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873bb0.
00000100:00000200:3.0:1788972397.856524:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8790000 x1875872189607808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972397.856532:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8790000 x1875872189607808/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972413 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972397.856550:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3f3e200.
02000000:00000010:3.0:1788972397.856556:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d200.
00000100:00000010:3.0:1788972397.856560:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100.
02000000:00000010:3.0:1788972397.856564:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3f3e600.
02000000:00000010:3.0:1788972397.856566:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d800.
00000100:00000010:3.0:1788972397.856568:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793480.
02000000:00000010:3.0:1788972397.856571:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c3f3f200.
02000000:00000010:3.0:1788972397.856575:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d600.
00000100:00000010:3.0:1788972397.856577:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000.
00000100:00000010:3.0:1788972398.866179:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09ae0.
00000100:00000010:2.0F:1788972398.869534:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09a80.
00000001:02000400:3.0:1788972400.305475:0:118088:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 311: Fail bulk put in send wait completion ========================================================== 12:46:38 (1788972398)
00000001:02000400:1.0F:1788972402.246647:0:118226:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_311 Need kfi network type
00000100:00000010:2.0:1788972402.962727:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f800.
02000000:00000010:2.0:1788972402.962768:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f800.
00000100:00000010:2.0:1788972402.962833:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ce00.
02000000:00000010:2.0:1788972402.962839:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f600.
00000100:00000010:2.0:1788972402.962847:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480.
02000000:00000010:2.0:1788972402.962851:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f000.
02000000:00000010:2.0:1788972402.969998:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600.
00000400:00000200:2.0:1788972402.970020:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972402.970026:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01261b8.
00000400:00000010:2.0:1788972402.970035:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b198.
00000100:00000200:2.0:1788972402.970044:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189607936, portal 25
00000400:00000010:2.0:1788972402.970051:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b770.
00000100:00000200:2.0:1788972402.970054:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189607936, offset 0
00000400:00000200:2.0:1788972402.970065:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972402.970075:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972402.970276:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788972402.970283:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972402.970288:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788972402.970292:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972402.970296:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972402.970300:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [741:741:256:4294967295] 192.168.203.150@tcp LPNI seq info [741:741:8:4294967295]
00000400:00000200:2.0:1788972402.970305:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972402.970316:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972402.970323:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972402.970329:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0fa00.
00000800:00000200:2.0:1788972402.970337:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972402.970346:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972402.970352:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0fa00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972402.970411:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788972402.970418:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e2a0.
00000100:00000010:2.0:1788972402.970434:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e1e0.
02000000:00000010:2.0:1788972402.970455:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835000.
00000400:00000200:2.0:1788972402.970460:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972402.970462:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b58.
00000400:00000010:2.0:1788972402.970466:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7baa0.
00000100:00000200:2.0:1788972402.970469:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189608192, portal 10
00000400:00000010:2.0:1788972402.970475:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7ba18.
00000100:00000200:2.0:1788972402.970477:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189608192, offset 0
00000400:00000200:2.0:1788972402.970481:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972402.970485:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972402.970488:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788972402.970491:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972402.970494:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788972402.970496:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972402.970498:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972402.970501:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [742:742:255:4294967295] 192.168.203.150@tcp LPNI seq info [742:742:7:4294967295]
00000400:00000200:2.0:1788972402.970505:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972402.970511:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972402.970515:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972402.970518:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0ff00.
00000800:00000200:2.0:1788972402.970522:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972402.970526:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972402.970529:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0ff00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972402.970534:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788972402.970556:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8834800.
00000400:00000200:2.0:1788972402.970559:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972402.970562:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126ec8.
00000400:00000010:2.0:1788972402.970565:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b440.
00000100:00000200:2.0:1788972402.970567:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189608064, portal 10
00000400:00000010:2.0:1788972402.970572:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bc38.
00000100:00000200:2.0:1788972402.970574:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189608064, offset 0
00000400:00000200:2.0:1788972402.970578:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972402.970586:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972402.970589:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788972402.970592:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972402.970594:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788972402.970597:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972402.970599:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972402.970601:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [743:743:254:4294967295] 192.168.203.150@tcp LPNI seq info [743:743:6:4294967295]
00000400:00000200:2.0:1788972402.970605:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972402.970611:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972402.970615:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972402.970618:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f900.
00000800:00000200:2.0:1788972402.970622:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972402.970626:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972402.970628:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972402.970633:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788972402.977240:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972402.977254:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0fa00.
00000400:00000200:3.0:1788972402.977262:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972402.977273:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972402.977282:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b770
00000400:00000010:3.0:1788972402.977285:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b770.
00000100:00000200:3.0:1788972402.977299:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f800 x1875872189607936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972418 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972402.977331:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972402.977333:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0ff00.
00000400:00000200:3.0:1788972402.977337:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972402.977340:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972402.977343:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7ba18
00000400:00000010:3.0:1788972402.977345:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7ba18.
00000100:00000200:3.0:1788972402.977364:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f480 x1875872189608192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972402.980620:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972402.980632:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f900.
00000400:00000200:3.0:1788972402.980639:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972402.980650:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972402.980659:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bc38
00000400:00000010:3.0:1788972402.980662:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bc38.
00000100:00000200:3.0:1788972402.980675:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ce00 x1875872189608064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972402.980723:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.980728:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972402.980738:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981213:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981220:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981225:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887817000
00000400:00000200:3.0:1788972402.981230:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981237:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2455 [1] + 224
00000400:00000200:3.0:1788972402.981253:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972402.981257:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972402.981269:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f800 x1875872189607936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972402.981300:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981304:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972402.981309:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981311:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981314:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981317:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817100
00000400:00000200:3.0:1788972402.981320:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981325:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2465 [1] + 224
00000400:00000200:3.0:1788972402.981341:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972402.981345:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972402.981842:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f480 x1875872189608192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972402.981866:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981870:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972402.981874:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981877:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981880:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981883:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817080
00000400:00000200:3.0:1788972402.981887:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972402.981891:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2475 [1] + 224
00000400:00000200:3.0:1788972402.981903:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972402.981906:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972402.981913:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ce00 x1875872189608064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972402.981954:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126ec8.
00000400:00000200:3.0:1788972402.981960:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b440
00000400:00000010:3.0:1788972402.981962:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b440.
00000100:00000200:3.0:1788972402.981968:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ce00 x1875872189608064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972402.981978:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ce00 x1875872189608064/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972402.982014:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01261b8.
00000400:00000200:3.0:1788972402.982018:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b198
00000400:00000010:3.0:1788972402.982020:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b198.
00000100:00000200:3.0:1788972402.982025:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f800 x1875872189607936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972402.982032:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f800 x1875872189607936/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1788972402.982054:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b58.
00000400:00000200:3.0:1788972402.982057:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7baa0
00000400:00000010:3.0:1788972402.982058:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7baa0.
00000100:00000200:3.0:1788972402.982063:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f480 x1875872189608192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972402.982071:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f480 x1875872189608192/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972418 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972402.982216:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8834800.
02000000:00000010:3.0:1788972402.982225:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f600.
00000100:00000010:3.0:1788972402.982230:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ce00.
02000000:00000010:3.0:1788972402.982234:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600.
02000000:00000010:3.0:1788972402.982239:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f800.
00000100:00000010:3.0:1788972402.982241:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f800.
02000000:00000010:3.0:1788972402.982245:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835000.
02000000:00000010:3.0:1788972402.982247:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f000.
00000100:00000010:3.0:1788972402.982249:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480.
00000100:00000010:3.0:1788972403.986134:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e1e0.
00000100:00000010:2.0:1788972403.986266:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e2a0.
00000001:02000400:0.0F:1788972404.220843:0:118380:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 312: TAG_RX_OK is possible after TX_FAIL ========================================================== 12:46:42 (1788972402)
00000001:02000400:0.0:1788972405.734705:0:118520:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: SKIP: sanity-lnet test_312 Need kfi network type
00000001:02000400:2.0:1788972407.352356:0:118674:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 350: Check import --exec of ping/discover ========================================================== 12:46:46 (1788972406)
00000001:02000400:2.0:1788972407.436480:0:118680:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000100:00000010:3.0:1788972408.082281:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000.
02000000:00000010:3.0:1788972408.082310:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d800.
00000100:00000010:3.0:1788972408.082360:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793480.
02000000:00000010:3.0:1788972408.082366:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d600.
00000100:00000010:3.0:1788972408.082374:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793100.
02000000:00000010:3.0:1788972408.082381:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c518d100.
02000000:00000010:2.0:1788972408.082381:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d408da00.
00000400:00000200:2.0:1788972408.082399:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972408.082406:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126630.
00000400:00000010:2.0:1788972408.082413:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220.
02000000:00000010:3.0:1788972408.082418:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e0b1aa00.
00000100:00000200:2.0:1788972408.082421:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189608320, portal 25
00000400:00000010:2.0:1788972408.082429:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b3b8.
00000400:00000200:3.0:1788972408.082432:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000200:2.0:1788972408.082435:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189608320, offset 0
00000400:00000010:3.0:1788972408.082439:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10.
00000400:00000200:2.0:1788972408.082447:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000010:3.0:1788972408.082456:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873bb0.
00000400:00000200:2.0:1788972408.082458:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000200:3.0:1788972408.082468:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189608448, portal 10
00000400:00000200:2.0:1788972408.082474:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000010:3.0:1788972408.082478:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48735d8.
00000400:00000200:2.0:1788972408.082482:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788972408.082488:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000100:00000200:3.0:1788972408.082496:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189608448, offset 0
00000400:00000200:2.0:1788972408.082505:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1788972408.082511:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972408.082517:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788972408.082523:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [744:744:256:4294967295] 192.168.203.150@tcp LPNI seq info [744:744:8:4294967295]
00000400:00000200:2.0:1788972408.082529:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972408.082544:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972408.082552:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972408.082559:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000010:2.0:1788972408.082565:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f800.
00000400:00000200:3.0:1788972408.082571:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:3.0:1788972408.082578:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000800:00000200:2.0:1788972408.082578:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000400:00000200:3.0:1788972408.082587:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:3.0:1788972408.082594:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000800:00000200:2.0:1788972408.082600:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:3.0:1788972408.082602:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000800:00000200:2.0:1788972408.082612:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1788972408.082614:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [745:745:255:4294967295] 192.168.203.150@tcp LPNI seq info [745:745:7:4294967295]
00000400:00000200:3.0:1788972408.082623:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972408.082637:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788972408.082643:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e960.
00000100:00000010:2.0:1788972408.082657:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651eba0.
02000000:00000010:2.0:1788972408.082871:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d408d400.
00000400:00000200:2.0:1788972408.082876:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972408.082880:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b58.
00000400:00000010:2.0:1788972408.082885:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788972408.082889:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189608576, portal 10
00000400:00000010:2.0:1788972408.082894:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788972408.082896:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189608576, offset 0
00000400:00000200:2.0:1788972408.082902:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:3.0:1788972408.083223:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1788972408.083233:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788972408.083239:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000010:3.0:1788972408.083242:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c518d500.
00000400:00000200:2.0:1788972408.083250:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000800:00000200:3.0:1788972408.083256:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000400:00000200:2.0:1788972408.083261:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000800:00000200:3.0:1788972408.083269:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000400:00000200:2.0:1788972408.083270:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000800:00000200:3.0:1788972408.083277:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c518d500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972408.083281:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788972408.083289:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:3.0:1788972408.083294:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972408.083303:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [746:746:254:4294967295] 192.168.203.150@tcp LPNI seq info [746:746:6:4294967295]
00000400:00000200:2.0:1788972408.083311:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788972408.083327:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788972408.083347:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788972408.083353:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f600.
00000800:00000200:2.0:1788972408.083359:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788972408.083365:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788972408.083369:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788972408.083378:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788972408.083431:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972408.083435:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f800.
00000400:00000200:2.0:1788972408.083440:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972408.083445:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788972408.083451:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b3b8
00000400:00000010:2.0:1788972408.083454:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b3b8.
00000100:00000200:2.0:1788972408.083463:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8790000 x1875872189608320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972424 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788972408.083488:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972408.083491:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c518d500.
00000400:00000200:2.0:1788972408.083495:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972408.083499:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788972408.083502:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48735d8
00000400:00000010:2.0:1788972408.083504:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48735d8.
00000100:00000200:2.0:1788972408.083509:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793480 x1875872189608448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788972408.083575:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788972408.083578:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f600.
00000400:00000200:2.0:1788972408.083581:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788972408.083584:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788972408.083587:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550
00000400:00000010:2.0:1788972408.083588:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788972408.083593:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c8793100 x1875872189608576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972408.087415:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087427:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972408.087437:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087443:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087449:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087453:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817200
00000400:00000200:3.0:1788972408.087460:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087468:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2495 [1] + 224
00000400:00000200:3.0:1788972408.087481:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972408.087486:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972408.087503:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793480 x1875872189608448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972408.087533:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087551:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972408.087556:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087558:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087561:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087563:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887817180
00000400:00000200:3.0:1788972408.087566:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087570:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2485 [1] + 224
00000400:00000200:3.0:1788972408.087576:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972408.087579:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972408.087586:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8790000 x1875872189608320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972408.087618:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087622:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972408.087628:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087631:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087634:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972408.087639:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817280
00000400:00000200:3.0:1788972408.087642:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788972408.087648:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126630.
00000400:00000200:2.0:1788972408.087668:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220
00000400:00000010:2.0:1788972408.087671:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220.
00000400:00000200:3.0:1788972408.087679:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x24a5 [1] + 224
00000100:00000200:2.0:1788972408.087689:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8790000 x1875872189608320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972408.087696:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972408.087711:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788972408.087718:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8790000 x1875872189608320/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972408.087725:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c8793100 x1875872189608576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972408.087783:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d408da00.
02000000:00000010:2.0:1788972408.087796:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d800.
00000400:00000010:3.0:1788972408.087799:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10.
00000100:00000010:2.0:1788972408.087803:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000.
00000400:00000200:3.0:1788972408.087807:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873bb0
00000400:00000010:3.0:1788972408.087812:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873bb0.
00000100:00000200:3.0:1788972408.087849:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793480 x1875872189608448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1788972408.087860:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793480 x1875872189608448/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1788972408.087902:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e0b1aa00.
02000000:00000010:3.0:1788972408.087909:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d600.
00000100:00000010:3.0:1788972408.087915:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793480.
00000400:00000010:2.0:1788972408.088141:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b58.
00000400:00000200:2.0:1788972408.088146:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68
00000400:00000010:2.0:1788972408.088148:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68.
00000100:00000200:2.0:1788972408.088156:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c8793100 x1875872189608576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788972408.088167:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c8793100 x1875872189608576/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972424 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788972408.088191:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d408d400.
02000000:00000010:2.0:1788972408.088195:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c518d100.
00000100:00000010:2.0:1788972408.088199:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793100.
00000001:02000400:2.0:1788972408.692986:0:118816:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_350: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:3.0:1788972409.108571:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651eba0.
00000100:00000010:2.0:1788972409.111686:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e960.
Debug log: 478 lines, 478 kept, 0 dropped, 0 bad.