00000100:00000010:3.0F:1790701575.064134:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074ca80.
02000000:00000010:3.0:1790701575.064163:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441d00.
00000100:00000010:3.0:1790701575.064195:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074d880.
02000000:00000010:3.0:1790701575.064201:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441000.
00000100:00000010:3.0:1790701575.064210:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cd074c700.
02000000:00000010:3.0:1790701575.064214:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cc4441b00.
02000000:00000010:3.0:1790701575.064247:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056c00.
00000400:00000200:3.0:1790701575.064258:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701575.064263:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223f78.
00000400:00000010:3.0:1790701575.064270:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0605d8.
00000100:00000200:3.0:1790701575.064279:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537429120, portal 25
00000400:00000010:3.0:1790701575.064286:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060220.
00000100:00000200:3.0:1790701575.064289:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537429120, offset 0
00000400:00000200:3.0:1790701575.064299:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701575.064309:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701575.064318:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701575.064324:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701575.064328:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701575.064332:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701575.064335:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701575.064339:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [70:70:256:4294967295] 192.168.204.157@tcp LPNI seq info [70:70:8:4294967295]
00000400:00000200:3.0:1790701575.064345:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701575.064356:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701575.064364:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701575.064369:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441300.
00000800:00000200:3.0:1790701575.064377:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701575.064385:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701575.064390:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441300 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701575.064403:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000100:00000010:3.0:1790701575.064408:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934dea0.
00000100:00000010:3.0:1790701575.064419:0:123347:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d060.
02000000:00000010:3.0:1790701575.064429:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1057800.
00000400:00000200:3.0:1790701575.064439:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701575.064442:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223e18.
00000400:00000010:3.0:1790701575.064446:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060088.
00000100:00000200:3.0:1790701575.064449:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537429376, portal 10
00000400:00000010:3.0:1790701575.064454:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060880.
00000100:00000200:3.0:1790701575.064456:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537429376, offset 0
00000400:00000200:3.0:1790701575.064461:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701575.064465:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701575.064468:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701575.064471:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701575.064473:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701575.064475:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701575.064478:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701575.064482:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [71:71:255:4294967295] 192.168.204.157@tcp LPNI seq info [71:71:7:4294967295]
00000400:00000200:3.0:1790701575.064485:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701575.064490:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701575.064495:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701575.064499:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441600.
00000800:00000200:3.0:1790701575.064502:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701575.064507:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701575.064509:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701575.064516:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
02000000:00000010:3.0:1790701575.064560:0:123347:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd1056800.
00000400:00000200:3.0:1790701575.064565:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701575.064567:0:123347:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62238f0.
00000400:00000010:3.0:1790701575.064570:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060e58.
00000100:00000200:3.0:1790701575.064573:0:123347:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537429248, portal 10
00000400:00000010:3.0:1790701575.064578:0:123347:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0603b8.
00000100:00000200:3.0:1790701575.064580:0:123347:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537429248, offset 0
00000400:00000200:3.0:1790701575.064585:0:123347:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701575.064588:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701575.064597:0:123347:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701575.064600:0:123347:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701575.064602:0:123347:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701575.064604:0:123347:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701575.064606:0:123347:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701575.064609:0:123347:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [72:72:254:4294967295] 192.168.204.157@tcp LPNI seq info [72:72:6:4294967295]
00000400:00000200:3.0:1790701575.064612:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701575.064618:0:123347:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701575.064623:0:123347:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701575.064626:0:123347:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441200.
00000800:00000200:3.0:1790701575.064630:0:123347:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701575.064634:0:123347:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701575.064636:0:123347:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701575.064641:0:123347:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000800:00000200:3.0:1790701575.064673:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701575.064676:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441300.
00000400:00000200:3.0:1790701575.064681:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701575.064685:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701575.064690:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060220
00000400:00000010:3.0:1790701575.064692:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060220.
00000100:00000200:3.0:1790701575.064700:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074ca80 x1877686537429120/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1790701575.064721:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701575.064723:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441600.
00000400:00000200:3.0:1790701575.064725:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701575.064728:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701575.064730:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060880
00000400:00000010:3.0:1790701575.064732:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060880.
00000100:00000200:3.0:1790701575.064736:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074c700 x1877686537429376/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1790701575.064784:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1790701575.064804:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441200.
00000400:00000200:3.0:1790701575.064807:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701575.064811:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:3.0:1790701575.064814:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0603b8
00000400:00000010:3.0:1790701575.064842:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0603b8.
00000100:00000200:3.0:1790701575.064847:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cd074d880 x1877686537429248/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0F:1790701575.083737:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083758:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:2.0:1790701575.083768:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083773:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083779:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083782:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102080
00000400:00000200:2.0:1790701575.083799:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083806:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x4dd [1] + 224
00000400:00000200:2.0:1790701575.083843:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701575.083848:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:2.0:1790701575.083864:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074ca80 x1877686537429120/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1790701575.083898:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083901:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:2.0:1790701575.083905:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083908:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083910:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083913:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102180
00000400:00000200:2.0:1790701575.083916:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083919:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x4ed [1] + 224
00000400:00000200:2.0:1790701575.083925:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701575.083928:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:2.0:1790701575.083934:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074c700 x1877686537429376/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1790701575.083950:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083953:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:2.0:1790701575.083957:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083959:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083961:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083964:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102100
00000400:00000200:2.0:1790701575.083966:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701575.083977:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x4fd [1] + 224
00000400:00000200:2.0:1790701575.083987:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701575.083989:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:2.0:1790701575.083995:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cd074d880 x1877686537429248/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701575.090814:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62238f0.
00000400:00000200:3.0:1790701575.090945:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060e58
00000400:00000010:3.0:1790701575.090947:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060e58.
00000100:00000200:3.0:1790701575.090961:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074d880 x1877686537429248/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701575.090973:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074d880 x1877686537429248/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701575.091004:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223f78.
00000400:00000200:3.0:1790701575.091006:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0605d8
00000400:00000010:3.0:1790701575.091008:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0605d8.
00000100:00000200:3.0:1790701575.091012:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074ca80 x1877686537429120/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701575.091021:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074ca80 x1877686537429120/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701575.091032:0:123347:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223e18.
00000400:00000200:3.0:1790701575.091034:0:123347:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060088
00000400:00000010:3.0:1790701575.091035:0:123347:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060088.
00000100:00000200:3.0:1790701575.091040:0:123347:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cd074c700 x1877686537429376/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701575.091048:0:123347:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cd074c700 x1877686537429376/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701591 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701575.091063:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056800.
02000000:00000010:3.0:1790701575.091084:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441000.
00000100:00000010:3.0:1790701575.091089:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074d880.
02000000:00000010:3.0:1790701575.091093:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1056c00.
02000000:00000010:3.0:1790701575.091095:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441d00.
00000100:00000010:3.0:1790701575.091097:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074ca80.
02000000:00000010:3.0:1790701575.091100:0:123347:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd1057800.
02000000:00000010:3.0:1790701575.091103:0:123347:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cc4441b00.
00000100:00000010:3.0:1790701575.091105:0:123347:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cd074c700.
00000400:00000010:3.0:1790701575.437316:0:123341:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at ffff8f8cc318e540.
00000400:00000010:3.0:1790701575.437327:0:123341:0:(lib-lnet.h:456:lnet_rspt_free()) free 'rspt': 64 at ffff8f8cc7e06a00.
00000100:00000010:3.0:1790701576.396140:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d060.
00000100:00000010:2.0:1790701576.396484:0:120878:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934dea0.
00000001:02000400:3.0:1790701578.080099:0:126926:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 476: lnetctl set latency_stats toggles capture ========================================================== 13:06:16 (1790701576)
00000001:02000400:2.0:1790701578.217413:0:126932:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:0.0F:1790701579.716740:0:127069:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_476: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:2.0:1790701580.175789:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671180.
02000000:00000010:2.0:1790701580.175812:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24fd00.
00000100:00000010:2.0:1790701580.175879:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca672300.
02000000:00000010:2.0:1790701580.175885:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f100.
00000100:00000010:2.0:1790701580.175892:0:71452:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8f8cca671f80.
02000000:00000010:2.0:1790701580.175896:0:71452:0:(sec_null.c:135:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8f8cfe24f600.
02000000:00000010:3.0:1790701580.176562:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd10cea00.
00000400:00000200:3.0:1790701580.176575:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701580.176580:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223e18.
00000400:00000010:3.0:1790701580.176587:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060088.
00000100:00000200:3.0:1790701580.176595:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537429504, portal 25
00000400:00000010:3.0:1790701580.176611:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0605d8.
00000100:00000200:3.0:1790701580.176615:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1877686537429504, offset 0
00000400:00000200:3.0:1790701580.176625:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701580.176634:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.176642:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701580.176647:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701580.176651:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701580.176655:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701580.176658:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701580.176661:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [73:73:256:4294967295] 192.168.204.157@tcp LPNI seq info [73:73:8:4294967295]
00000400:00000200:3.0:1790701580.176666:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.176686:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701580.176693:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701580.176698:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441800.
00000800:00000200:3.0:1790701580.176719:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701580.176727:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701580.176731:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701580.176750:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000100:00000010:3.0:1790701580.176756:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d000.
00000100:00000010:3.0:1790701580.176770:0:123346:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8f8cc934d5a0.
02000000:00000010:3.0:1790701580.176781:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd10cec00.
00000400:00000200:3.0:1790701580.176784:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701580.176787:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf6223f78.
00000400:00000010:3.0:1790701580.176790:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060e58.
00000100:00000200:3.0:1790701580.176793:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537429760, portal 10
00000400:00000010:3.0:1790701580.176798:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe0603b8.
00000100:00000200:3.0:1790701580.176801:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537429760, offset 0
00000400:00000200:3.0:1790701580.176805:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701580.176808:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.176812:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701580.176814:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701580.176833:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701580.176836:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701580.176838:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701580.176840:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [74:74:255:4294967295] 192.168.204.157@tcp LPNI seq info [74:74:7:4294967295]
00000400:00000200:3.0:1790701580.176844:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.176850:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701580.176854:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701580.176858:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441b00.
00000800:00000200:3.0:1790701580.176861:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701580.176865:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701580.176868:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441b00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701580.176874:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
02000000:00000010:3.0:1790701580.176928:0:123346:0:(sec_null.c:178:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8f8cd10cf600.
00000400:00000200:3.0:1790701580.176932:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:3.0:1790701580.176940:0:123346:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8f8cf62238f0.
00000400:00000010:3.0:1790701580.176944:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060880.
00000100:00000200:3.0:1790701580.176947:0:123346:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1877686537429632, portal 10
00000400:00000010:3.0:1790701580.176952:0:123346:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8f8cfe060220.
00000100:00000200:3.0:1790701580.176954:0:123346:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1877686537429632, offset 0
00000400:00000200:3.0:1790701580.176959:0:123346:0:(lib-move.c:5150:LNetPut()) LNetPut -> 12345-192.168.204.157@tcp
00000400:00000200:3.0:1790701580.176962:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.176966:0:123346:0:(lib-move.c:2806:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.204.157@tcp local destination
00000400:00000200:3.0:1790701580.176968:0:123346:0:(lib-move.c:2437:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:3.0:1790701580.176971:0:123346:0:(lib-move.c:1618:lnet_get_best_ni()) selected best_ni 192.168.204.57@tcp
00000400:00000200:3.0:1790701580.176973:0:123346:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.204.57@tcp lpni_is_preferred = 0
00000400:00000200:3.0:1790701580.176975:0:123346:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.157@tcp
00000400:00000200:3.0:1790701580.176977:0:123346:0:(lib-move.c:1738:lnet_handle_send()) 192.168.204.57@tcp NI seq info: [75:75:254:4294967295] 192.168.204.157@tcp LPNI seq info [75:75:6:4294967295]
00000400:00000200:3.0:1790701580.176981:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.176987:0:123346:0:(lib-move.c:1854:lnet_handle_send()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp:<?>) -> 192.168.204.157@tcp(192.168.204.157@tcp:192.168.204.157@tcp) <?> : PUT try# 0
00000800:00000200:3.0:1790701580.176991:0:123346:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.157@tcp
00000800:00000010:3.0:1790701580.176994:0:123346:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8f8cc4441d00.
00000800:00000200:3.0:1790701580.176999:0:123346:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8f8ce1b22480] -> 12345-192.168.204.157@tcp (5)
00000800:00000200:3.0:1790701580.177003:0:123346:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.157@tcp ip 192.168.204.157:988
00000800:00000200:3.0:1790701580.177005:0:123346:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8f8cc4441d00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:3.0:1790701580.177010:0:123346:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000800:00000200:2.0:1790701580.177113:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1790701580.177117:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441800.
00000400:00000200:2.0:1790701580.177123:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701580.177129:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:2.0:1790701580.177134:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0605d8
00000400:00000010:2.0:1790701580.177137:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0605d8.
00000100:00000200:2.0:1790701580.177146:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cca671180 x1877686537429504/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1790701580.177167:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1790701580.177170:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441b00.
00000400:00000200:2.0:1790701580.177172:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701580.177175:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:2.0:1790701580.177177:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe0603b8
00000400:00000010:2.0:1790701580.177178:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe0603b8.
00000100:00000200:2.0:1790701580.177183:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cca671f80 x1877686537429760/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1790701580.177251:0:123338:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1790701580.177259:0:123338:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8f8cc4441d00.
00000400:00000200:2.0:1790701580.177263:0:123338:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1790701580.177266:0:123338:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000400:00000200:2.0:1790701580.177269:0:123338:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060220
00000400:00000010:2.0:1790701580.177271:0:123338:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060220.
00000100:00000200:2.0:1790701580.177276:0:123338:0:(events.c:40:request_out_callback()) @@@ type 5, status 0  req@ffff8f8cca672300 x1877686537429632/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701580.183385:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.183396:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:3.0:1790701580.183406:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.183411:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.183417:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.183421:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 25 MB=0x6abbef7102200
00000400:00000200:3.0:1790701580.183427:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.183433:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.157@tcp of length 224/224 into md 0x50d [1] + 224
00000400:00000200:3.0:1790701580.183456:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701580.183462:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701580.187631:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cca671180 x1877686537429504/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701580.187695:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.187701:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
00000400:00000200:3.0:1790701580.187707:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:2.0:1790701580.187708:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223e18.
00000400:00000200:3.0:1790701580.187711:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701580.187714:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060088
00000400:00000200:3.0:1790701580.187716:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000010:2.0:1790701580.187716:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060088.
00000400:00000200:3.0:1790701580.187719:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102300
00000100:00000200:2.0:1790701580.187723:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cca671180 x1877686537429504/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701580.187724:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.187729:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x51d [1] + 224
00000100:00000200:2.0:1790701580.187732:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cca671180 x1877686537429504/t0(0) o400->MGC192.168.204.157@tcp@192.168.204.157@tcp:26/25 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701580.187739:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701580.187742:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701580.187747:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cca671f80 x1877686537429760/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1790701580.187758:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
02000000:00000010:2.0:1790701580.187758:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd10cea00.
00000400:00000200:3.0:1790701580.187761:0:123339:0:(lib-move.c:4645:lnet_parse()) TRACE: 192.168.204.57@tcp(192.168.204.57@tcp) <- 192.168.204.157@tcp : PUT - for me
02000000:00000010:2.0:1790701580.187763:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24fd00.
00000400:00000200:3.0:1790701580.187765:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.187767:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000100:00000010:2.0:1790701580.187768:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671180.
00000400:00000200:3.0:1790701580.187769:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:3.0:1790701580.187771:0:123339:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.204.157@tcp of length 224 into portal 10 MB=0x6abbef7102280
00000400:00000010:2.0:1790701580.187774:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf6223f78.
00000400:00000200:3.0:1790701580.187775:0:123339:0:(api-ni.c:1619:lnet_nid4_cpt_hash()) Match nid 192.168.204.157@tcp to cpt 1
00000400:00000200:2.0:1790701580.187776:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060e58
00000400:00000010:2.0:1790701580.187778:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060e58.
00000100:00000200:2.0:1790701580.187781:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cca671f80 x1877686537429760/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1790701580.187795:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cca671f80 x1877686537429760/t0(0) o400->lustre-MDT0001-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1790701580.187815:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd10cec00.
02000000:00000010:2.0:1790701580.187844:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f600.
00000100:00000010:2.0:1790701580.187848:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca671f80.
00000400:00000200:3.0:1790701580.193921:0:123339:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.157@tcp of length 224/224 into md 0x52d [1] + 224
00000400:00000200:3.0:1790701580.193958:0:123339:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1790701580.193963:0:123339:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.204.57@tcp->192.168.204.157@tcp: PUT: OK
00000100:00000200:3.0:1790701580.193977:0:123339:0:(events.c:82:reply_in_callback()) @@@ type 2, status 0  req@ffff8f8cca672300 x1877686537429632/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:3.0:1790701580.194025:0:123346:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8f8cf62238f0.
00000400:00000200:3.0:1790701580.194029:0:123346:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8f8cfe060880
00000400:00000010:3.0:1790701580.194032:0:123346:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8f8cfe060880.
00000100:00000200:3.0:1790701580.194046:0:123346:0:(events.c:82:reply_in_callback()) @@@ type 6, status 0  req@ffff8f8cca672300 x1877686537429632/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:3.0:1790701580.194055:0:123346:0:(events.c:104:reply_in_callback()) @@@ unlink  req@ffff8f8cca672300 x1877686537429632/t0(0) o400->lustre-MDT0000-mdc-ffff8f8cc7fce800@192.168.204.157@tcp:12/10 lens 224/224 e 0 to 0 dl 1790701596 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:3.0:1790701580.194091:0:123346:0:(sec_null.c:192:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8f8cd10cf600.
02000000:00000010:3.0:1790701580.194111:0:123346:0:(sec_null.c:162:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8f8cfe24f100.
00000100:00000010:3.0:1790701580.194116:0:123346:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8f8cca672300.
00000100:00000010:3.0:1790701581.451879:0:112539:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d5a0.
00000100:00000010:2.0:1790701581.451879:0:120878:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8f8cc934d000.
Debug log: 319 lines, 319 kept, 0 dropped, 0 bad.