00000100:00000010:3.0F:1790701590.416674:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c700.
02000000:00000010:3.0:1790701590.416698:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441000.
00000100:00000010:3.0:1790701590.416727:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ca80.
02000000:00000010:3.0:1790701590.416731:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441d00.
00000100:00000010:3.0:1790701590.416737:0:121163:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d880.
02000000:00000010:3.0:1790701590.416741:0:121163:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441b00.
02000000:00000010:3.0:1790701590.416777:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bce00.
00000400:00000200:3.0:1790701590.416787:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701590.416791:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62238f0.
00000400:00000010:3.0:1790701590.416797:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060880.
00000100:00000200:3.0:1790701590.416805:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537430272, portal 25
00000400:00000010:3.0:1790701590.416810:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060110.
00000100:00000200:3.0:1790701590.416812:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537430272, offset 0
00000400:00000200:3.0:1790701590.416839:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701590.416847:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.416855:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701590.416860:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701590.416865:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701590.416869:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701590.416872:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701590.416875:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [79:79:256:4294967295] 192.168.204.157@tcp LPNI seq info [79:79:8:4294967295]
00000400:00000200:3.0:1790701590.416880:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.416889:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701590.416896:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701590.416900:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441800.
00000800:00000200:3.0:1790701590.416907:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701590.416915:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701590.416920:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701590.416932:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000100:00000010:3.0:1790701590.416936:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d4e0.
00000100:00000010:3.0:1790701590.416947:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d000.
02000000:00000010:3.0:1790701590.416956:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bd200.
00000400:00000200:3.0:1790701590.416964:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701590.416967:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223688.
00000400:00000010:3.0:1790701590.416970:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060f68.
00000100:00000200:3.0:1790701590.416973:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537430528, portal 10
00000400:00000010:3.0:1790701590.416978:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060770.
00000100:00000200:3.0:1790701590.416980:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537430528, offset 0
00000400:00000200:3.0:1790701590.416985:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701590.416988:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.416992:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701590.416995:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701590.416997:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701590.416999:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701590.417002:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701590.417005:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [80:80:255:4294967295] 192.168.204.157@tcp LPNI seq info [80:80:7:4294967295]
00000400:00000200:3.0:1790701590.417009:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.417015:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701590.417020:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701590.417023:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441100.
00000800:00000200:3.0:1790701590.417027:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701590.417031:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701590.417033:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701590.417038:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
02000000:00000010:3.0:1790701590.417053:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc52bd400.
00000400:00000200:3.0:1790701590.417057:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701590.417059:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62239a0.
00000400:00000010:3.0:1790701590.417061:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060aa0.
00000100:00000200:3.0:1790701590.417064:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537430400, portal 10
00000400:00000010:3.0:1790701590.417068:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0607f8.
00000100:00000200:3.0:1790701590.417070:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537430400, offset 0
00000400:00000200:3.0:1790701590.417074:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701590.417077:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.417083:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701590.417086:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701590.417088:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701590.417090:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701590.417092:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701590.417095:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [81:81:254:4294967295] 192.168.204.157@tcp LPNI seq info [81:81:6:4294967295]
00000400:00000200:3.0:1790701590.417099:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.417105:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701590.417109:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701590.417112:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441e00.
00000800:00000200:3.0:1790701590.417115:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701590.417119:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701590.417122:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701590.417126:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000800:00000200:3.0:1790701590.417168:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701590.417171:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441800.
00000400:00000200:3.0:1790701590.417175:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701590.417180:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701590.417185:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060110
00000400:00000010:3.0:1790701590.417188:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060110.
00000100:00000200:3.0:1790701590.417199:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074c700 x1877686537430272/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1790701590.417381:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701590.417384:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441100.
00000400:00000200:3.0:1790701590.417388:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701590.417391:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701590.417394:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060770
00000400:00000010:3.0:1790701590.417395:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060770.
00000100:00000200:3.0:1790701590.417400:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074d880 x1877686537430528/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1790701590.417462:0:123339:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701590.417469:0:123339:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441e00.
00000400:00000200:3.0:1790701590.417655:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701590.417659:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701590.417662:0:123339:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0607f8
00000400:00000010:3.0:1790701590.417664:0:123339:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0607f8.
00000100:00000200:3.0:1790701590.417670:0:123339:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074ca80 x1877686537430400/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701590.420097:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.420108:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:3.0:1790701590.420116:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.420120:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.420125:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.420129:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102580
00000400:00000200:3.0:1790701590.420134:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701590.420140:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x575 [1] + 224
00000400:00000200:3.0:1790701590.420153:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701590.420157:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701590.420171:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074ca80 x1877686537430400/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701590.420214:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62239a0.
00000400:00000200:3.0:1790701590.420218:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060aa0
00000400:00000010:3.0:1790701590.420220:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060aa0.
00000100:00000200:3.0:1790701590.420225:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074ca80 x1877686537430400/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701590.420233:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074ca80 x1877686537430400/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701590.420284:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bd400.
02000000:00000010:3.0:1790701590.420290:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441d00.
00000100:00000010:3.0:1790701590.420295:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ca80.
00000400:00000200:2.0F:1790701590.445410:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445428:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:2.0:1790701590.445437:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445443:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445449:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445452:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102600
00000400:00000200:2.0:1790701590.445458:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445464:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x565 [1] + 224
00000400:00000200:2.0:1790701590.445476:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701590.445480:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:2.0:1790701590.445497:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074d880 x1877686537430528/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701590.445538:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223688.
00000400:00000200:2.0:1790701590.445542:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060f68
00000400:00000010:2.0:1790701590.445545:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060f68.
00000100:00000200:2.0:1790701590.445550:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074d880 x1877686537430528/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701590.445558:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074d880 x1877686537430528/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701590.445593:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bd200.
02000000:00000010:2.0:1790701590.445599:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441b00.
00000100:00000010:2.0:1790701590.445603:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d880.
00000400:00000200:2.0:1790701590.445621:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445624:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:2.0:1790701590.445630:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445632:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445635:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445639:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102500
00000400:00000200:2.0:1790701590.445642:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701590.445646:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x555 [1] + 224
00000400:00000200:2.0:1790701590.445655:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701590.445663:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:2.0:1790701590.445670:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074c700 x1877686537430272/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701590.445689:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62238f0.
00000400:00000200:2.0:1790701590.445692:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060880
00000400:00000010:2.0:1790701590.445694:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060880.
00000100:00000200:2.0:1790701590.445698:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074c700 x1877686537430272/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701590.445706:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074c700 x1877686537430272/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701606 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701590.445719:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc52bce00.
02000000:00000010:2.0:1790701590.445722:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441000.
00000100:00000010:2.0:1790701590.445725:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c700.
00000100:00000010:3.0:1790701591.755912:0:125766:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d000.
00000100:00000010:2.0:1790701591.760805:0:103943:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d4e0.
00000001:02000400:1.0F:1790701591.895897:0:128109:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 501: Verify lnetctl peer set --health (SR) ========================================================== 13:06:30 (1790701590)
00000001:02000400:3.0:1790701592.200145:0:128123:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:0.0F:1790701593.623005:0:128251:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_501: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:3.0:1790701595.534580:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ca80.
02000000:00000010:3.0:1790701595.534605:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441e00.
00000100:00000010:3.0:1790701595.534637:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074f800.
02000000:00000010:3.0:1790701595.534641:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441d00.
00000100:00000010:3.0:1790701595.534647:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c380.
02000000:00000010:3.0:1790701595.534650:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441800.
02000000:00000010:3.0:1790701595.534682:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc9b4d000.
00000400:00000200:3.0:1790701595.534692:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701595.534697:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62239a0.
00000400:00000010:3.0:1790701595.534704:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060aa0.
00000100:00000200:3.0:1790701595.534713:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537430656, portal 25
00000400:00000010:3.0:1790701595.534720:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0607f8.
00000100:00000200:3.0:1790701595.534722:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537430656, offset 0
00000400:00000200:3.0:1790701595.534732:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701595.534740:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.534748:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701595.534754:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701595.534758:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701595.534763:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701595.534766:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701595.534770:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [82:82:256:4294967295] 192.168.204.157@tcp LPNI seq info [82:82:8:4294967295]
00000400:00000200:3.0:1790701595.534775:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.534784:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701595.534791:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701595.534796:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441f00.
00000800:00000200:3.0:1790701595.534803:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701595.534811:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701595.534842:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441f00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701595.534856:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000100:00000010:3.0:1790701595.534869:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934dde0.
00000100:00000010:3.0:1790701595.534880:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d0c0.
02000000:00000010:3.0:1790701595.534892:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc9b4d600.
00000400:00000200:3.0:1790701595.534896:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701595.534899:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223318.
00000400:00000010:3.0:1790701595.534903:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060770.
00000100:00000200:3.0:1790701595.534906:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537430912, portal 10
00000400:00000010:3.0:1790701595.534911:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060110.
00000100:00000200:3.0:1790701595.534914:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537430912, offset 0
00000400:00000200:3.0:1790701595.534918:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701595.534921:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.534925:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701595.534927:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701595.534929:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701595.534932:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701595.534934:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701595.534937:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [83:83:255:4294967295] 192.168.204.157@tcp LPNI seq info [83:83:7:4294967295]
00000400:00000200:3.0:1790701595.534941:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.534947:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701595.534951:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701595.534955:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441700.
00000800:00000200:3.0:1790701595.534959:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701595.534963:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701595.534978:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441700 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701595.534985:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
02000000:00000010:3.0:1790701595.535005:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cc9b4d800.
00000400:00000200:3.0:1790701595.535008:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701595.535011:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62231b8.
00000400:00000010:3.0:1790701595.535014:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060ee0.
00000100:00000200:3.0:1790701595.535017:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537430784, portal 10
00000400:00000010:3.0:1790701595.535021:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060a18.
00000100:00000200:3.0:1790701595.535029:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537430784, offset 0
00000400:00000200:3.0:1790701595.535033:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701595.535037:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.535040:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701595.535043:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701595.535045:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701595.535047:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701595.535049:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701595.535051:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [84:84:254:4294967295] 192.168.204.157@tcp LPNI seq info [84:84:6:4294967295]
00000400:00000200:3.0:1790701595.535055:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.535061:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701595.535066:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701595.535069:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441500.
00000800:00000200:3.0:1790701595.535073:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701595.535077:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701595.535080:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441500 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701595.535085:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000800:00000200:3.0:1790701595.535130:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701595.535135:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441f00.
00000400:00000200:3.0:1790701595.535140:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701595.535156:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701595.535162:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0607f8
00000400:00000010:3.0:1790701595.535165:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0607f8.
00000100:00000200:3.0:1790701595.535176:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074ca80 x1877686537430656/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1790701595.535201:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701595.535203:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441700.
00000400:00000200:3.0:1790701595.535206:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701595.535209:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701595.535212:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060110
00000400:00000010:3.0:1790701595.535214:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060110.
00000100:00000200:3.0:1790701595.535225:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074c380 x1877686537430912/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1790701595.535296:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701595.535299:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441500.
00000400:00000200:3.0:1790701595.535302:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701595.535306:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701595.535308:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060a18
00000400:00000010:3.0:1790701595.535310:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060a18.
00000100:00000200:3.0:1790701595.535315:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074f800 x1877686537430784/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701595.543352:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543366:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:3.0:1790701595.543377:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543382:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543388:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543392:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102780
00000400:00000200:3.0:1790701595.543398:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543405:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x595 [1] + 224
00000400:00000200:3.0:1790701595.543417:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701595.543422:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701595.543439:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074c380 x1877686537430912/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701595.543471:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543487:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:3.0:1790701595.543492:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543495:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543498:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543501:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102700
00000400:00000200:3.0:1790701595.543504:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.543508:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x5a5 [1] + 224
00000400:00000200:3.0:1790701595.543543:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701595.543548:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701595.543556:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074f800 x1877686537430784/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701595.543592:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62231b8.
00000400:00000200:3.0:1790701595.543596:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060ee0
00000400:00000010:3.0:1790701595.543599:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060ee0.
00000100:00000200:3.0:1790701595.543605:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074f800 x1877686537430784/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701595.543613:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074f800 x1877686537430784/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701595.543646:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223318.
00000400:00000200:3.0:1790701595.543649:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060770
00000400:00000010:3.0:1790701595.543651:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060770.
00000100:00000200:3.0:1790701595.543655:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074c380 x1877686537430912/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701595.543664:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074c380 x1877686537430912/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701595.543684:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc9b4d800.
02000000:00000010:3.0:1790701595.543692:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441d00.
00000100:00000010:3.0:1790701595.543698:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074f800.
02000000:00000010:3.0:1790701595.543702:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc9b4d600.
02000000:00000010:3.0:1790701595.543705:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441800.
00000100:00000010:3.0:1790701595.543708:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c380.
00000400:00000200:3.0:1790701595.546924:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.546935:0:123338:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:3.0:1790701595.546944:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.546949:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.546955:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.547080:0:123338:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102680
00000400:00000200:3.0:1790701595.547085:0:123338:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701595.547092:0:123338:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x585 [1] + 224
00000400:00000200:3.0:1790701595.547108:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701595.547113:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701595.547129:0:123338:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074ca80 x1877686537430656/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1790701595.547227:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62239a0.
00000400:00000200:2.0:1790701595.547234:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060aa0
00000400:00000010:2.0:1790701595.547236:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060aa0.
00000100:00000200:2.0:1790701595.547243:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074ca80 x1877686537430656/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701595.547253:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074ca80 x1877686537430656/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701611 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701595.547283:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cc9b4d000.
02000000:00000010:2.0:1790701595.547288:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441e00.
00000100:00000010:2.0:1790701595.547293:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ca80.
Debug log: 315 lines, 315 kept, 0 dropped, 0 bad.