00000100:00000010:2.0F:1788971819.283174:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ca80.
02000000:00000010:2.0:1788971819.283203:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63200.
00000100:00000010:2.0:1788971819.283231:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000.
02000000:00000010:2.0:1788971819.283235:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63100.
00000100:00000010:2.0:1788971819.283241:0:108765:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f480.
02000000:00000010:2.0:1788971819.283246:0:108765:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0a4c600.
02000000:00000010:2.0:1788971819.283272:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd200.
00000400:00000200:2.0:1788971819.283281:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971819.283286:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126840.
00000400:00000010:2.0:1788971819.283292:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b220.
00000100:00000200:2.0:1788971819.283298:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564288, portal 10
00000400:00000010:2.0:1788971819.283304:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58.
00000100:00000200:2.0:1788971819.283307:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189564288, offset 0
00000400:00000200:2.0:1788971819.283315:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971819.283323:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971819.283331:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971819.283336:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971819.283341:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971819.283344:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971819.283347:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971819.283351:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [399:399:256:4294967295] 192.168.203.150@tcp LPNI seq info [399:399:8:4294967295]
00000400:00000200:2.0:1788971819.283355:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971819.283364:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971819.283370:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971819.283375:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0a4c100.
00000800:00000200:2.0:1788971819.283381:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971819.283389:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971819.283394:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0a4c100 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971819.283408:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:2.0:1788971819.283412:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e240.
00000100:00000010:2.0:1788971819.283423:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e120.
02000000:00000010:2.0:1788971819.283443:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd400.
00000400:00000200:2.0:1788971819.283454:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971819.283457:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126d10.
00000400:00000010:2.0:1788971819.283460:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880.
00000100:00000200:2.0:1788971819.283464:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564160, portal 25
00000400:00000010:2.0:1788971819.283468:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000.
00000100:00000200:2.0:1788971819.283470:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189564160, offset 0
00000400:00000200:2.0:1788971819.283475:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971819.283479:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971819.283482:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971819.283486:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971819.283488:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971819.283490:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971819.283493:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971819.283495:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [400:400:255:4294967295] 192.168.203.150@tcp LPNI seq info [400:400:7:4294967295]
00000400:00000200:2.0:1788971819.283499:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971819.283505:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971819.283510:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971819.283514:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383e00.
00000800:00000200:2.0:1788971819.283519:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971819.283523:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971819.283525:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383e00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971819.283530:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:2.0:1788971819.283542:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00.
00000400:00000200:2.0:1788971819.283546:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:2.0:1788971819.283548:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126e18.
00000400:00000010:2.0:1788971819.283552:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788971819.283554:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564416, portal 10
00000400:00000010:2.0:1788971819.283559:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788971819.283561:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189564416, offset 0
00000400:00000200:2.0:1788971819.283566:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:2.0:1788971819.283569:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971819.283581:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:2.0:1788971819.283584:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:2.0:1788971819.283587:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:2.0:1788971819.283589:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:2.0:1788971819.283592:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:2.0:1788971819.283595:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [401:401:254:4294967295] 192.168.203.150@tcp LPNI seq info [401:401:6:4294967295]
00000400:00000200:2.0:1788971819.283599:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971819.283605:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:2.0:1788971819.283610:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:2.0:1788971819.283614:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c5383a00.
00000800:00000200:2.0:1788971819.283619:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:2.0:1788971819.283623:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:2.0:1788971819.283626:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c5383a00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:2.0:1788971819.283631:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971819.283675:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971819.283679:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0a4c100.
00000400:00000200:2.0:1788971819.283687:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971819.283692:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971819.283698:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58
00000400:00000010:2.0:1788971819.283700:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58.
00000100:00000200:2.0:1788971819.283715:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840c000 x1875872189564288/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971819.283739:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971819.283741:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383e00.
00000400:00000200:2.0:1788971819.283744:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971819.283747:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971819.283750:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000
00000400:00000010:2.0:1788971819.283751:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000.
00000100:00000200:2.0:1788971819.283755:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840ca80 x1875872189564160/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971819.283910:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971819.283918:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c5383a00.
00000400:00000200:2.0:1788971819.283922:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971819.283927:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971819.283931:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8
00000400:00000010:2.0:1788971819.283933:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8.
00000100:00000200:2.0:1788971819.283948:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c840f480 x1875872189564416/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0F:1788971819.287634:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287650:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971819.287658:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287662:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287667:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287671:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780c500
00000400:00000200:3.0:1788971819.287677:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287682:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x13b5 [1] + 224
00000400:00000200:3.0:1788971819.287693:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971819.287697:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971819.287709:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840ca80 x1875872189564160/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971819.287740:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287744:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971819.287747:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287750:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287752:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287755:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780c580
00000400:00000200:3.0:1788971819.287758:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287762:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x13a5 [1] + 224
00000400:00000200:3.0:1788971819.287767:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971819.287770:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971819.287776:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840c000 x1875872189564288/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:3.0:1788971819.287787:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287790:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:3.0:1788971819.287793:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287795:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287798:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287800:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780c600
00000400:00000200:3.0:1788971819.287803:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:3.0:1788971819.287812:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x13c5 [1] + 224
00000400:00000200:3.0:1788971819.287851:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:3.0:1788971819.287867:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:3.0:1788971819.287873:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c840f480 x1875872189564416/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971819.288393:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126d10.
00000400:00000200:2.0:1788971819.288400:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880
00000400:00000010:2.0:1788971819.288403:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880.
00000100:00000200:2.0:1788971819.288412:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840ca80 x1875872189564160/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971819.288422:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840ca80 x1875872189564160/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971819.288451:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126e18.
00000400:00000200:2.0:1788971819.288454:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550
00000400:00000010:2.0:1788971819.288456:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550.
00000100:00000200:2.0:1788971819.288461:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840f480 x1875872189564416/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971819.288469:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840f480 x1875872189564416/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:2.0:1788971819.288484:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126840.
00000400:00000200:2.0:1788971819.288487:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b220
00000400:00000010:2.0:1788971819.288488:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b220.
00000100:00000200:2.0:1788971819.288492:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c840c000 x1875872189564288/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:2.0:1788971819.288499:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c840c000 x1875872189564288/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971835 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:2.0:1788971819.288515:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd400.
02000000:00000010:2.0:1788971819.288528:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63200.
00000100:00000010:2.0:1788971819.288533:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ca80.
02000000:00000010:2.0:1788971819.288537:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00.
02000000:00000010:2.0:1788971819.288540:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0a4c600.
00000100:00000010:2.0:1788971819.288544:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f480.
02000000:00000010:2.0:1788971819.288546:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd200.
02000000:00000010:2.0:1788971819.288548:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63100.
00000100:00000010:2.0:1788971819.288550:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000.
00000100:00000010:3.0:1788971820.306010:0:56197:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e120.
00000100:00000010:2.0:1788971820.307315:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e240.
00000001:02000400:3.0:1788971822.177450:0:111688:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 291: load lnet using custom accept port for bulk tcp connections ========================================================== 12:37:00 (1788971820)
00000001:02000400:3.0:1788971822.383567:0:111696:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure
00000001:02000400:3.0:1788971823.922259:0:111831:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_291: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240
00000100:00000010:1.0F:1788971824.411438:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480.
02000000:00000010:1.0:1788971824.411475:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f800.
00000100:00000010:1.0:1788971824.411515:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700.
02000000:00000010:1.0:1788971824.411521:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0fa00.
00000100:00000010:1.0:1788971824.411527:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80.
02000000:00000010:1.0:1788971824.411530:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c8a0f000.
02000000:00000010:1.0:1788971824.411561:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ec00.
00000400:00000200:1.0:1788971824.411570:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971824.411574:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000010:1.0:1788971824.411581:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660.
00000100:00000200:1.0:1788971824.411588:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564672, portal 10
00000400:00000010:1.0:1788971824.411594:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525908.
00000100:00000200:1.0:1788971824.411597:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189564672, offset 0
00000400:00000200:1.0:1788971824.411606:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971824.411616:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971824.411625:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971824.411630:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971824.411635:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971824.411638:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971824.411641:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971824.411645:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [402:402:256:4294967295] 192.168.203.150@tcp LPNI seq info [402:402:8:4294967295]
00000400:00000200:1.0:1788971824.411649:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971824.411659:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971824.411665:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971824.411670:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0ff00.
00000800:00000200:1.0:1788971824.411676:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971824.411684:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971824.411688:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0ff00 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971824.411710:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000100:00000010:1.0:1788971824.411714:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467b40.
00000100:00000010:1.0:1788971824.411724:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74673c0.
02000000:00000010:1.0:1788971824.411743:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fc00.
00000400:00000200:1.0:1788971824.411754:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971824.411756:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000010:1.0:1788971824.411760:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38.
00000100:00000200:1.0:1788971824.411762:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564544, portal 25
00000400:00000010:1.0:1788971824.411766:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525990.
00000100:00000200:1.0:1788971824.411768:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189564544, offset 0
00000400:00000200:1.0:1788971824.411772:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971824.411776:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971824.411780:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971824.411783:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971824.411785:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971824.411787:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971824.411790:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971824.411792:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [403:403:255:4294967295] 192.168.203.150@tcp LPNI seq info [403:403:7:4294967295]
00000400:00000200:1.0:1788971824.411796:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971824.411843:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971824.411848:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971824.411852:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f600.
00000800:00000200:1.0:1788971824.411855:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971824.411859:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971824.411862:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f600 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971824.411866:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
02000000:00000010:1.0:1788971824.411878:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600.
00000400:00000200:1.0:1788971824.411881:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000010:1.0:1788971824.411883:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528.
00000400:00000010:1.0:1788971824.411886:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58.
00000100:00000200:1.0:1788971824.411888:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189564800, portal 10
00000400:00000010:1.0:1788971824.411892:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770.
00000100:00000200:1.0:1788971824.411895:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189564800, offset 0
00000400:00000200:1.0:1788971824.411899:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp
00000400:00000200:1.0:1788971824.411903:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971824.411922:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR:  192.168.203.150@tcp local destination
00000400:00000200:1.0:1788971824.411925:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp
00000400:00000200:1.0:1788971824.411927:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp
00000400:00000200:1.0:1788971824.411929:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0
00000400:00000200:1.0:1788971824.411931:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp
00000400:00000200:1.0:1788971824.411934:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [404:404:254:4294967295] 192.168.203.150@tcp LPNI seq info [404:404:6:4294967295]
00000400:00000200:1.0:1788971824.411937:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:1.0:1788971824.411943:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:<?>) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) <?> : PUT try# 0
00000800:00000200:1.0:1788971824.411948:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp
00000800:00000010:1.0:1788971824.411952:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c8a0f900.
00000800:00000200:1.0:1788971824.411956:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5)
00000800:00000200:1.0:1788971824.411960:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988
00000800:00000200:1.0:1788971824.411962:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c8a0f900 type 1, nob 320 niov 1 nkiov 1
00000400:00000200:1.0:1788971824.411968:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000800:00000200:2.0:1788971824.412205:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971824.412209:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0ff00.
00000400:00000200:2.0:1788971824.412214:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971824.412219:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971824.412225:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525908
00000400:00000010:2.0:1788971824.412227:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525908.
00000100:00000200:2.0:1788971824.412237:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f0700 x1875872189564672/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971824.412260:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971824.412263:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f600.
00000400:00000200:2.0:1788971824.412265:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971824.412268:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971824.412271:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525990
00000400:00000010:2.0:1788971824.412272:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525990.
00000100:00000200:2.0:1788971824.412276:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f3480 x1875872189564544/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000800:00000200:2.0:1788971824.412341:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0
00000800:00000010:2.0:1788971824.412344:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c8a0f900.
00000400:00000200:2.0:1788971824.412347:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971824.412350:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000400:00000200:2.0:1788971824.412352:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770
00000400:00000010:2.0:1788971824.412354:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770.
00000100:00000200:2.0:1788971824.412358:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0  req@ffff9a73c87f2d80 x1875872189564800/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971824.431614:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431624:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971824.431633:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431638:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431643:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431647:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780c700
00000400:00000200:2.0:1788971824.431653:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431659:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x13d5 [1] + 224
00000400:00000200:2.0:1788971824.431670:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971824.431675:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971824.431688:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f0700 x1875872189564672/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971824.431770:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431773:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971824.431777:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431780:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431783:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431785:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa188780c780
00000400:00000200:2.0:1788971824.431789:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431793:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x13e5 [1] + 224
00000400:00000200:2.0:1788971824.431799:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971824.431828:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971824.431835:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f2d80 x1875872189564800/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000200:2.0:1788971824.431851:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431854:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me
00000400:00000200:2.0:1788971824.431858:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431860:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431872:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431875:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa188780c680
00000400:00000200:2.0:1788971824.431878:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1
00000400:00000200:2.0:1788971824.431882:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x13dd [1] + 224
00000400:00000200:2.0:1788971824.431894:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1788971824.431897:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK
00000100:00000200:2.0:1788971824.431914:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0  req@ffff9a73c87f3480 x1875872189564544/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971824.439633:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d5d8.
00000400:00000200:1.0:1788971824.439643:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660
00000400:00000010:1.0:1788971824.439646:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660.
00000100:00000200:1.0:1788971824.439664:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f0700 x1875872189564672/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971824.439675:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f0700 x1875872189564672/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971824.439709:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ec00.
02000000:00000010:1.0:1788971824.439716:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0fa00.
00000100:00000010:1.0:1788971824.439721:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700.
00000400:00000010:1.0:1788971824.439729:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8.
00000400:00000200:1.0:1788971824.439731:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38
00000400:00000010:1.0:1788971824.439732:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38.
00000100:00000200:1.0:1788971824.439737:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f3480 x1875872189564544/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971824.439744:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f3480 x1875872189564544/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000400:00000010:1.0:1788971824.439765:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528.
00000400:00000200:1.0:1788971824.439768:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58
00000400:00000010:1.0:1788971824.439769:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58.
00000100:00000200:1.0:1788971824.439774:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0  req@ffff9a73c87f2d80 x1875872189564800/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
00000100:00000200:1.0:1788971824.439782:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink  req@ffff9a73c87f2d80 x1875872189564800/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788971840 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295
02000000:00000010:1.0:1788971824.439793:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fc00.
02000000:00000010:1.0:1788971824.439797:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f800.
00000100:00000010:1.0:1788971824.439799:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480.
02000000:00000010:1.0:1788971824.439802:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600.
02000000:00000010:1.0:1788971824.439804:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c8a0f000.
00000100:00000010:1.0:1788971824.439806:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80.
00000100:00000010:3.0:1788971825.426891:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74673c0.
00000100:00000010:2.0:1788971825.427072:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467b40.
Debug log: 317 lines, 317 kept, 0 dropped, 0 bad.