00000100:00000010:2.0F:1788972479.766940:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972479.766966:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387300. 00000100:00000010:2.0:1788972479.766994:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972479.766999:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387d00. 00000100:00000010:2.0:1788972479.767004:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972479.767008:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d1387900. 02000000:00000010:2.0:1788972479.767799:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f600. 00000400:00000200:2.0:1788972479.767840:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972479.767845:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126528. 00000400:00000010:2.0:1788972479.767853:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972479.767859:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189613952, portal 25 00000400:00000010:2.0:1788972479.767864:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972479.767867:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189613952, offset 0 00000400:00000200:2.0:1788972479.767877:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972479.767885:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.767893:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972479.767899:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972479.767903:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972479.767907:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972479.767910:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972479.767913:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [788:788:256:4294967295] 192.168.203.150@tcp LPNI seq info [788:788:8:4294967295] 00000400:00000200:2.0:1788972479.767918:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.767928:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972479.767935:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972479.767940:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387100. 00000800:00000200:2.0:1788972479.767946:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972479.767953:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972479.767958:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972479.767973:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972479.767978:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e240. 00000100:00000010:2.0:1788972479.767989:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651eae0. 02000000:00000010:2.0:1788972479.768002:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9fa00. 00000400:00000200:2.0:1788972479.768016:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972479.768019:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:2.0:1788972479.768023:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972479.768026:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189614208, portal 10 00000400:00000010:2.0:1788972479.768031:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788972479.768033:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189614208, offset 0 00000400:00000200:2.0:1788972479.768038:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972479.768042:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.768045:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972479.768048:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972479.768051:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972479.768053:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972479.768056:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972479.768059:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [789:789:255:4294967295] 192.168.203.150@tcp LPNI seq info [789:789:7:4294967295] 00000400:00000200:2.0:1788972479.768063:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.768069:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972479.768074:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972479.768078:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387700. 00000800:00000200:2.0:1788972479.768082:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972479.768086:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972479.768089:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972479.768505:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972479.768527:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0a9f000. 00000400:00000200:2.0:1788972479.768531:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972479.768534:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972479.768538:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972479.768541:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189614080, portal 10 00000400:00000010:2.0:1788972479.768546:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972479.768548:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189614080, offset 0 00000400:00000200:2.0:1788972479.768553:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972479.768557:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.768564:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972479.768568:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972479.768570:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972479.768572:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972479.768574:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972479.768577:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [790:790:254:4294967295] 192.168.203.150@tcp LPNI seq info [790:790:6:4294967295] 00000400:00000200:2.0:1788972479.768581:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.768587:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972479.768592:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972479.768595:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d1387800. 00000800:00000200:2.0:1788972479.768599:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972479.768604:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972479.768607:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d1387800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972479.768612:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972479.768660:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972479.768664:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387100. 00000400:00000200:2.0:1788972479.768669:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972479.768673:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972479.768678:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972479.768681:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972479.768689:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189613952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972495 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972479.768710:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972479.768713:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387700. 00000400:00000200:2.0:1788972479.768715:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972479.768718:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972479.768720:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:2.0:1788972479.768722:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788972479.768726:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189614208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972479.768802:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972479.768810:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d1387800. 00000400:00000200:2.0:1788972479.768838:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972479.768842:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972479.768846:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788972479.768847:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972479.768853:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189614080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972479.772210:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772219:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972479.772227:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772232:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772237:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772241:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818880 00000400:00000200:2.0:1788972479.772247:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772262:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x26cd [1] + 224 00000400:00000200:2.0:1788972479.772273:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972479.772277:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972479.772291:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189614208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972479.772320:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772324:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972479.772328:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772330:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772333:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772336:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818800 00000400:00000200:2.0:1788972479.772339:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772342:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x26dd [1] + 224 00000400:00000200:2.0:1788972479.772349:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972479.772353:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972479.772359:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189614080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972479.772390:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772394:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972479.772397:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772400:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772402:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772405:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887818780 00000400:00000200:2.0:1788972479.772408:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972479.772412:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x26bd [1] + 224 00000400:00000200:2.0:1788972479.772418:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972479.772421:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972479.772427:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189613952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972479.772451:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972479.772455:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:2.0:1788972479.772457:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972479.772463:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189614080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972479.772471:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189614080/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972479.772496:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126528. 00000400:00000200:2.0:1788972479.772498:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:2.0:1788972479.772500:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972479.772504:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189613952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972479.772510:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189613952/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972479.772519:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:2.0:1788972479.772520:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:2.0:1788972479.772527:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972479.772532:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189614208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972479.772540:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189614208/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972495 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972479.772556:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f000. 02000000:00000010:2.0:1788972479.772561:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387d00. 00000100:00000010:2.0:1788972479.772566:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972479.772569:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9f600. 02000000:00000010:2.0:1788972479.772571:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387300. 00000100:00000010:2.0:1788972479.772573:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972479.772575:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0a9fa00. 02000000:00000010:2.0:1788972479.772577:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d1387900. 00000100:00000010:2.0:1788972479.772578:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 00000100:00000010:3.0F:1788972480.785898:0:116657:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651eae0. 00000100:00000010:2.0:1788972480.789611:0:116364:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e240. 00000001:02000400:2.0:1788972482.892955:0:124731:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 502: Verify lnetctl peer set --health (MR) ========================================================== 12:48:01 (1788972481) 00000001:02000400:0.0F:1788972483.051193:0:124742:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: /home/green/git/lustre-release/lustre/../lnet/utils/lnetctl lnet unconfigure 00000001:02000400:0.0:1788972484.213681:0:124873:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_502: @@@@@@ FAIL: lnetctl lnet unconfigure failed 240 00000100:00000010:1.0F:1788972484.889864:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800. 02000000:00000010:1.0:1788972484.889887:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0d00. 00000100:00000010:1.0:1788972484.889924:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788972484.889933:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0100. 00000100:00000010:1.0:1788972484.889943:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1f80. 02000000:00000010:1.0:1788972484.889947:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73e04c0c00. 02000000:00000010:1.0:1788972484.889975:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b800. 00000400:00000200:1.0:1788972484.889984:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972484.889988:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972484.889994:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788972484.890001:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189614336, portal 25 00000400:00000010:1.0:1788972484.890006:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972484.890009:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189614336, offset 0 00000400:00000200:1.0:1788972484.890017:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972484.890025:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972484.890032:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972484.890037:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972484.890040:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972484.890044:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972484.890047:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972484.890051:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [791:791:256:4294967295] 192.168.203.150@tcp LPNI seq info [791:791:8:4294967295] 00000400:00000200:1.0:1788972484.890055:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972484.890064:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972484.890070:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972484.890075:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0200. 00000800:00000200:1.0:1788972484.890081:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972484.890088:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972484.890093:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972484.890110:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972484.890120:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c38e3f60. 00000100:00000010:1.0:1788972484.890129:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c38e36c0. 02000000:00000010:1.0:1788972484.890139:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a400. 00000400:00000200:1.0:1788972484.890148:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972484.890150:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972484.890154:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525cc0. 00000100:00000200:1.0:1788972484.890157:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189614592, portal 10 00000400:00000010:1.0:1788972484.890162:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525b28. 00000100:00000200:1.0:1788972484.890164:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189614592, offset 0 00000400:00000200:1.0:1788972484.890169:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972484.890173:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972484.890177:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972484.890180:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972484.890182:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972484.890184:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972484.890187:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972484.890190:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [792:792:255:4294967295] 192.168.203.150@tcp LPNI seq info [792:792:7:4294967295] 00000400:00000200:1.0:1788972484.890194:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972484.890200:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972484.890204:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972484.890207:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0000. 00000800:00000200:1.0:1788972484.890211:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972484.890226:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972484.890230:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972484.890236:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972484.890252:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094b400. 00000400:00000200:1.0:1788972484.890256:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972484.890258:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d6e0. 00000400:00000010:1.0:1788972484.890261:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788972484.890263:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189614464, portal 10 00000400:00000010:1.0:1788972484.890267:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788972484.890269:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189614464, offset 0 00000400:00000200:1.0:1788972484.890273:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972484.890277:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972484.890285:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972484.890288:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972484.890291:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972484.890293:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972484.890295:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972484.890298:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [793:793:254:4294967295] 192.168.203.150@tcp LPNI seq info [793:793:6:4294967295] 00000400:00000200:1.0:1788972484.890302:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972484.890308:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972484.890313:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972484.890316:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73e04c0400. 00000800:00000200:1.0:1788972484.890320:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972484.890324:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972484.890327:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73e04c0400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972484.890332:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972484.890531:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972484.890536:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0200. 00000400:00000200:2.0:1788972484.890542:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972484.890548:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972484.890552:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18 00000400:00000010:2.0:1788972484.890555:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18. 00000100:00000200:2.0:1788972484.890564:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3800 x1875872189614336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972500 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972484.890585:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972484.890588:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0000. 00000400:00000200:2.0:1788972484.890590:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972484.890593:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972484.890595:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525b28 00000400:00000010:2.0:1788972484.890597:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525b28. 00000100:00000200:2.0:1788972484.890608:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1f80 x1875872189614592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972484.890666:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972484.890670:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73e04c0400. 00000400:00000200:2.0:1788972484.890673:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972484.890676:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972484.890679:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525110 00000400:00000010:2.0:1788972484.890681:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525110. 00000100:00000200:2.0:1788972484.890685:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189614464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972484.896273:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896282:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972484.896290:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896295:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896300:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896305:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818a00 00000400:00000200:2.0:1788972484.896311:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896316:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x26f5 [1] + 224 00000400:00000200:2.0:1788972484.896327:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972484.896333:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972484.896413:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1f80 x1875872189614592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972484.896495:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896499:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972484.896504:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896506:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896509:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896512:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887818980 00000400:00000200:2.0:1788972484.896515:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972484.896519:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x26fd [1] + 224 00000400:00000200:2.0:1788972484.896535:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972484.896539:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972484.896545:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189614464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972484.897954:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972484.897964:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525cc0 00000400:00000010:1.0:1788972484.897967:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525cc0. 00000100:00000200:1.0:1788972484.897978:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1f80 x1875872189614592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972484.897988:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1f80 x1875872189614592/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972484.898020:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a400. 02000000:00000010:1.0:1788972484.898029:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0c00. 00000100:00000010:1.0:1788972484.898034:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1f80. 00000400:00000010:1.0:1788972484.898042:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d6e0. 00000400:00000200:1.0:1788972484.898044:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220 00000400:00000010:1.0:1788972484.898046:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788972484.898050:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189614464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972484.898058:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189614464/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972484.898082:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b400. 02000000:00000010:1.0:1788972484.898087:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0100. 00000100:00000010:1.0:1788972484.898090:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 00000400:00000200:3.0:1788972484.898434:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972484.898440:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972484.898448:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972484.898452:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972484.898457:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972484.898461:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887818900 00000400:00000200:3.0:1788972484.898466:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972484.898472:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x26ed [1] + 224 00000400:00000200:3.0:1788972484.898485:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972484.898489:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972484.898499:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3800 x1875872189614336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972484.899068:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972484.899074:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68 00000400:00000010:1.0:1788972484.899075:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788972484.899083:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3800 x1875872189614336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972484.899093:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3800 x1875872189614336/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972500 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972484.899111:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094b800. 02000000:00000010:1.0:1788972484.899116:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73e04c0d00. 00000100:00000010:1.0:1788972484.899119:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800. Debug log: 315 lines, 315 kept, 0 dropped, 0 bad.