00000100:00000010:1.0F:1788972433.691152:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80.
02000000:00000010:1.0:1788972433.691188:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1be5d00.
00000100:00000010:1.0:1788972433.691229:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788972433.691234:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1be5800.
00000100:00000010:1.0:1788972433.691241:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80.
02000000:00000010:1.0:1788972433.691246:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1be5a00.
02000000:00000010:1.0:1788972433.691283:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cee400.
00000400:00000200:1.0:1788972433.691294:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788972433.691298:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8.
00000400:00000010:1.0:1788972433.691305:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525990.
00000100:00000200:1.0:1788972433.691312:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189610496, portal 25
00000400:00000010:1.0:1788972433.691316:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38.
00000100:00000200:1.0:1788972433.691319:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189610496, offset 0
00000400:00000200:1.0:1788972433.691329:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788972433.691337:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972433.691345:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788972433.691350:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972433.691354:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788972433.691357:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972433.691360:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788972433.691363:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [761:761:256:4294967295] 192.168.203.150@tcp LPNI seq info [761:761:8:4294967295]
00000400:00000200:1.0:1788972433.691367:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972433.691376:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972433.691382:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788972433.691388:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0400.
00000800:00000200:1.0:1788972433.691394:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788972433.691401:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788972433.691405:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0400 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972433.691422:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:1.0:1788972433.691427:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74670c0.
00000100:00000010:1.0:1788972433.691437:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467720.
02000000:00000010:1.0:1788972433.691448:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cef800.
00000400:00000200:1.0:1788972433.691458:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788972433.691460:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000.
00000400:00000010:1.0:1788972433.691463:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000.
00000100:00000200:1.0:1788972433.691466:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189610752, portal 10
00000400:00000010:1.0:1788972433.691475:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770.
00000100:00000200:1.0:1788972433.691477:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189610752, offset 0
00000400:00000200:1.0:1788972433.691481:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788972433.691484:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972433.691488:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788972433.691490:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972433.691492:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788972433.691494:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972433.691497:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788972433.691499:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [762:762:255:4294967295] 192.168.203.150@tcp LPNI seq info [762:762:7:4294967295]
00000400:00000200:1.0:1788972433.691503:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972433.691508:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972433.691512:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788972433.691515:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0200.
00000800:00000200:1.0:1788972433.691518:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788972433.691522:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788972433.691525:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0200 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972433.691530:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:1.0:1788972433.691546:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c7cee600.
00000400:00000200:1.0:1788972433.691550:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788972433.691552:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d6e0.
00000400:00000010:1.0:1788972433.691555:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18.
00000100:00000200:1.0:1788972433.691557:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189610624, portal 10
00000400:00000010:1.0:1788972433.691561:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58.
00000100:00000200:1.0:1788972433.691563:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189610624, offset 0
00000400:00000200:1.0:1788972433.691567:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788972433.691572:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0F:1788972433.692936:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972433.692963:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0400.
00000400:00000200:3.0:1788972433.692972:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972433.692980:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972433.692987:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38
00000400:00000010:3.0:1788972433.692989:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38.
00000100:00000200:3.0:1788972433.693000:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f1f80 x1875872189610496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972449 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972433.693316:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972433.693320:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0200.
00000400:00000200:3.0:1788972433.693323:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972433.693327:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972433.693331:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770
00000400:00000010:3.0:1788972433.693333:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770.
00000100:00000200:3.0:1788972433.693339:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f2d80 x1875872189610752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:1.0:1788972433.696736:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788972433.696744:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972433.696749:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788972433.696753:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972433.696756:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788972433.696760:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [763:763:256:4294967295] 192.168.203.150@tcp LPNI seq info [763:763:8:4294967295]
00000400:00000200:1.0:1788972433.696767:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972433.696780:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972433.696787:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788972433.696794:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0900.
00000800:00000200:1.0:1788972433.696800:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788972433.696807:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788972433.696810:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972433.696845:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.697709:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.697718:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972433.697725:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.697729:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.697734:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.697739:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887817a00
00000400:00000200:3.0:1788972433.697745:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.697751:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x2555 [1] + 224
00000400:00000200:3.0:1788972433.697765:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972433.697770:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972433.697783:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f1f80 x1875872189610496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972433.698657:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972433.698663:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0900.
00000400:00000200:3.0:1788972433.698668:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972433.698674:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972433.698685:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58
00000400:00000010:3.0:1788972433.698688:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58.
00000100:00000200:3.0:1788972433.698696:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189610624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972433.698723:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.698726:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972433.698731:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.698733:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.698737:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.698741:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817b00
00000400:00000200:3.0:1788972433.698745:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.698750:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x255d [1] + 224
00000400:00000200:3.0:1788972433.698759:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972433.698763:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972433.698769:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f2d80 x1875872189610752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972433.699698:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.699704:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972433.699709:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.699712:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.699715:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.699718:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817a80
00000400:00000200:3.0:1788972433.699722:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972433.699727:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2565 [1] + 224
00000400:00000200:3.0:1788972433.701442:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972433.701453:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972433.701466:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189610624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972433.702689:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d6e0.
00000400:00000200:1.0:1788972433.702697:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18
00000400:00000010:1.0:1788972433.702699:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18.
00000100:00000200:1.0:1788972433.702723:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189610624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972433.702735:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189610624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972433.702767:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8.
00000400:00000200:1.0:1788972433.702771:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525990
00000400:00000010:1.0:1788972433.702774:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525990.
00000100:00000200:1.0:1788972433.702780:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f1f80 x1875872189610496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972433.702789:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f1f80 x1875872189610496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972433.702800:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000.
00000400:00000200:1.0:1788972433.702860:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000
00000400:00000010:1.0:1788972433.702861:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000.
00000100:00000200:1.0:1788972433.702868:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f2d80 x1875872189610752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972433.702877:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f2d80 x1875872189610752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972449 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788972433.702898:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cee600.
02000000:00000010:1.0:1788972433.702904:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1be5800.
00000100:00000010:1.0:1788972433.702909:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788972433.702914:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cee400.
02000000:00000010:1.0:1788972433.702917:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1be5d00.
00000100:00000010:1.0:1788972433.702920:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80.
02000000:00000010:1.0:1788972433.702923:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c7cef800.
02000000:00000010:1.0:1788972433.702926:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1be5a00.
00000100:00000010:1.0:1788972433.702928:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80.
00000100:00000010:3.0:1788972434.713318:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467720.
00000100:00000010:2.0F:1788972434.715963:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74670c0.
00000001:02000400:2.0:1788972435.884881:0:121039:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 403: Add and delete udsp net and nid rules ========================================================== 12:47:14 (1788972434)
00000001:02000400:3.0:1788972436.107214:0:121053:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:0.0F:1788972437.773514:0:121181:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_403: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:1.0:1788972438.804916:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80.
02000000:00000010:1.0:1788972438.804939:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0900.
00000100:00000010:1.0:1788972438.804969:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80.
02000000:00000010:1.0:1788972438.804972:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0200.
00000100:00000010:1.0:1788972438.804977:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800.
02000000:00000010:1.0:1788972438.804981:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0400.
02000000:00000010:1.0:1788972438.805012:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17af400.
00000400:00000200:1.0:1788972438.805060:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788972438.805064:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000.
00000400:00000010:1.0:1788972438.805071:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000.
00000100:00000200:1.0:1788972438.805077:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189610880, portal 25
00000400:00000010:1.0:1788972438.805082:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525990.
00000100:00000200:1.0:1788972438.805085:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189610880, offset 0
00000400:00000200:1.0:1788972438.805094:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788972438.805102:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972438.805109:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788972438.805125:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972438.805129:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788972438.805133:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972438.805136:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788972438.805140:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [764:764:256:4294967295] 192.168.203.150@tcp LPNI seq info [764:764:8:4294967295]
00000400:00000200:1.0:1788972438.805144:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972438.805154:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972438.805160:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788972438.805167:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1be5a00.
00000800:00000200:1.0:1788972438.805174:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788972438.805181:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788972438.805185:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1be5a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972438.805207:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:1.0:1788972438.805213:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467ea0.
00000100:00000010:1.0:1788972438.805225:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467a80.
02000000:00000010:1.0:1788972438.805236:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17af000.
00000400:00000200:1.0:1788972438.805240:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788972438.805242:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8.
00000400:00000010:1.0:1788972438.805245:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18.
00000100:00000200:1.0:1788972438.805248:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189611136, portal 10
00000400:00000010:1.0:1788972438.805253:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0.
00000100:00000200:1.0:1788972438.805256:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189611136, offset 0
00000400:00000200:1.0:1788972438.805261:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788972438.805264:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972438.805267:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788972438.805270:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972438.805273:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788972438.805275:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972438.805278:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788972438.805281:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [765:765:255:4294967295] 192.168.203.150@tcp LPNI seq info [765:765:7:4294967295]
00000400:00000200:1.0:1788972438.805285:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972438.805299:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972438.805304:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788972438.805307:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1be5d00.
00000800:00000200:1.0:1788972438.805312:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788972438.805317:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788972438.805320:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1be5d00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972438.805327:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:1.0:1788972438.805344:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17ae000.
00000400:00000200:1.0:1788972438.805348:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788972438.805350:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d6e0.
00000400:00000010:1.0:1788972438.805354:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68.
00000100:00000200:1.0:1788972438.805357:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189611008, portal 10
00000400:00000010:1.0:1788972438.805361:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8.
00000100:00000200:1.0:1788972438.805364:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189611008, offset 0
00000400:00000200:1.0:1788972438.805368:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788972438.805372:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972438.805375:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788972438.805378:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788972438.805381:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788972438.805383:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788972438.805385:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788972438.805388:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [766:766:254:4294967295] 192.168.203.150@tcp LPNI seq info [766:766:6:4294967295]
00000400:00000200:1.0:1788972438.805392:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788972438.805398:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788972438.805402:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788972438.805406:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1be5800.
00000800:00000200:1.0:1788972438.805409:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788972438.805414:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788972438.805416:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1be5800 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788972438.805421:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:3.0:1788972438.807939:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972438.807949:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1be5a00.
00000400:00000200:3.0:1788972438.807958:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972438.807965:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972438.807973:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525990
00000400:00000010:3.0:1788972438.807975:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525990.
00000100:00000200:3.0:1788972438.807987:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f2d80 x1875872189610880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972454 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972438.808011:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972438.808014:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1be5d00.
00000400:00000200:3.0:1788972438.808017:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972438.808020:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972438.808023:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0
00000400:00000010:3.0:1788972438.808025:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0.
00000100:00000200:3.0:1788972438.808030:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3800 x1875872189611136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:3.0:1788972438.808283:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:3.0:1788972438.808286:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1be5800.
00000400:00000200:3.0:1788972438.808290:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972438.808294:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:3.0:1788972438.808298:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8
00000400:00000010:3.0:1788972438.808300:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8.
00000100:00000200:3.0:1788972438.808306:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f1f80 x1875872189611008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972438.811976:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.811986:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972438.811994:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.811998:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812003:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812006:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887817b80
00000400:00000200:3.0:1788972438.812022:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812028:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x256d [1] + 224
00000400:00000200:3.0:1788972438.812038:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972438.812042:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972438.812055:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f2d80 x1875872189610880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972438.812083:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812086:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972438.812090:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812093:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812095:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812098:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817c80
00000400:00000200:3.0:1788972438.812105:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812109:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2575 [1] + 224
00000400:00000200:3.0:1788972438.812114:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972438.812117:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972438.812122:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3800 x1875872189611136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788972438.812133:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812137:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788972438.812140:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812142:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812145:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812147:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887817c00
00000400:00000200:3.0:1788972438.812150:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788972438.812153:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x257d [1] + 224
00000400:00000200:3.0:1788972438.812192:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788972438.812196:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788972438.812200:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f1f80 x1875872189611008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972438.813085:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d6e0.
00000400:00000200:1.0:1788972438.813092:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68
00000400:00000010:1.0:1788972438.813094:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68.
00000100:00000200:1.0:1788972438.813114:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f1f80 x1875872189611008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972438.813123:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f1f80 x1875872189611008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972438.813154:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000.
00000400:00000200:1.0:1788972438.813157:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000
00000400:00000010:1.0:1788972438.813159:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000.
00000100:00000200:1.0:1788972438.813163:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f2d80 x1875872189610880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972438.813171:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f2d80 x1875872189610880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788972438.813181:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8.
00000400:00000200:1.0:1788972438.813182:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18
00000400:00000010:1.0:1788972438.813184:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18.
00000100:00000200:1.0:1788972438.813187:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3800 x1875872189611136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788972438.813195:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3800 x1875872189611136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972454 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788972438.813208:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17ae000.
02000000:00000010:1.0:1788972438.813214:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0200.
00000100:00000010:1.0:1788972438.813219:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80.
02000000:00000010:1.0:1788972438.813223:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17af400.
02000000:00000010:1.0:1788972438.813225:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0900.
00000100:00000010:1.0:1788972438.813226:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80.
02000000:00000010:1.0:1788972438.813229:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17af000.
02000000:00000010:1.0:1788972438.813230:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0400.
00000100:00000010:1.0:1788972438.813232:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800.
00000100:00000010:3.0:1788972439.846421:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467a80.
00000100:00000010:2.0:1788972439.848161:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467ea0.
Debug log: 317 lines, 317 kept, 0 dropped, 0 bad.