00000100:00000010:1.0F:1788972116.242140:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788972116.242168:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b600. 00000100:00000010:1.0:1788972116.242197:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700. 02000000:00000010:1.0:1788972116.242202:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b400. 00000100:00000010:1.0:1788972116.242209:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972116.242214:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 02000000:00000010:1.0:1788972116.242242:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17af400. 00000400:00000200:1.0:1788972116.242291:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972116.242295:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d528. 00000400:00000010:1.0:1788972116.242303:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525aa0. 00000100:00000200:1.0:1788972116.242310:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586432, portal 25 00000400:00000010:1.0:1788972116.242315:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75255d8. 00000100:00000200:1.0:1788972116.242318:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189586432, offset 0 00000400:00000200:1.0:1788972116.242327:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972116.242335:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972116.242342:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972116.242405:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972116.242411:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972116.242415:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972116.242418:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972116.242422:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [573:573:256:4294967295] 192.168.203.150@tcp LPNI seq info [573:573:8:4294967295] 00000400:00000200:1.0:1788972116.242427:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972116.242437:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972116.242443:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972116.242448:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788972116.242455:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972116.242462:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972116.242466:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972116.242487:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972116.242492:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467ea0. 00000100:00000010:1.0:1788972116.242501:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467780. 02000000:00000010:1.0:1788972116.242512:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17afc00. 00000400:00000200:1.0:1788972116.242522:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972116.242524:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000010:1.0:1788972116.242528:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75253b8. 00000100:00000200:1.0:1788972116.242531:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586688, portal 10 00000400:00000010:1.0:1788972116.242536:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525bb0. 00000100:00000200:1.0:1788972116.242538:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189586688, offset 0 00000400:00000200:1.0:1788972116.242543:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972116.242955:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972116.242960:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972116.242963:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972116.242966:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972116.242968:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972116.242970:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972116.242973:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [574:574:255:4294967295] 192.168.203.150@tcp LPNI seq info [574:574:7:4294967295] 00000400:00000200:1.0:1788972116.242977:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972116.242983:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972116.242988:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972116.242992:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b000. 00000800:00000200:1.0:1788972116.242996:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972116.243001:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972116.243004:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b000 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:2.0F:1788972116.243007:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1788972116.243010:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:2.0:1788972116.243017:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:2.0:1788972116.243021:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972116.243026:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 02000000:00000010:1.0:1788972116.243029:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d17ae000. 00000400:00000200:2.0:1788972116.243031:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75255d8 00000400:00000200:1.0:1788972116.243032:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972116.243033:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75255d8. 00000400:00000010:1.0:1788972116.243035:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d898. 00000400:00000010:1.0:1788972116.243037:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525660. 00000100:00000200:1.0:1788972116.243040:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586560, portal 10 00000100:00000200:2.0:1788972116.243042:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189586432/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972116.243044:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75257f8. 00000100:00000200:1.0:1788972116.243047:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189586560, offset 0 00000400:00000200:1.0:1788972116.243051:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972116.243054:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972116.243062:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972116.243065:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:2.0:1788972116.243067:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1788972116.243068:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000800:00000010:2.0:1788972116.243069:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b000. 00000400:00000200:1.0:1788972116.243070:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972116.243072:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1788972116.243072:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972116.243075:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:1.0:1788972116.243075:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [575:575:255:4294967295] 192.168.203.150@tcp LPNI seq info [575:575:7:4294967295] 00000400:00000200:2.0:1788972116.243077:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525bb0 00000400:00000010:2.0:1788972116.243079:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525bb0. 00000400:00000200:1.0:1788972116.243079:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:2.0:1788972116.243083:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189586688/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972116.243086:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972116.243090:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972116.243093:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b900. 00000800:00000200:1.0:1788972116.243097:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972116.243101:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972116.243103:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972116.243125:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972116.243322:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972116.243325:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b900. 00000400:00000200:2.0:1788972116.243328:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972116.243332:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972116.243336:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75257f8 00000400:00000010:2.0:1788972116.243338:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75257f8. 00000100:00000200:2.0:1788972116.243343:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f0700 x1875872189586560/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972116.254474:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254484:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972116.254493:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254498:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254517:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254521:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811d00 00000400:00000200:2.0:1788972116.254526:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254532:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c4d [1] + 224 00000400:00000200:2.0:1788972116.254554:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972116.254560:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972116.254574:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189586688/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972116.254609:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254612:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972116.254616:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254618:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254621:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254623:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811c80 00000400:00000200:2.0:1788972116.254626:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254629:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c55 [1] + 224 00000400:00000200:2.0:1788972116.254634:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972116.254638:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972116.254643:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f0700 x1875872189586560/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972116.254656:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254659:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972116.254663:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254666:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254668:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254671:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811c00 00000400:00000200:2.0:1788972116.254673:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972116.254677:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c45 [1] + 224 00000400:00000200:2.0:1788972116.254684:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972116.254688:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972116.254693:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189586432/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0F:1788972116.254779:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d898. 00000400:00000200:0.0:1788972116.254790:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525660 00000400:00000010:0.0:1788972116.254792:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525660. 00000100:00000200:0.0:1788972116.254799:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f0700 x1875872189586560/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972116.254807:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f0700 x1875872189586560/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972116.254879:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d528. 00000400:00000200:0.0:1788972116.254883:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525aa0 00000400:00000010:0.0:1788972116.254885:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525aa0. 00000100:00000200:0.0:1788972116.254890:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189586432/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972116.254898:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189586432/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972116.254908:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d7e8. 00000400:00000200:0.0:1788972116.254910:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75253b8 00000400:00000010:0.0:1788972116.254912:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75253b8. 00000100:00000200:0.0:1788972116.254915:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189586688/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972116.254922:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189586688/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972132 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972116.254936:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17ae000. 02000000:00000010:0.0:1788972116.254942:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b400. 00000100:00000010:0.0:1788972116.254947:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700. 02000000:00000010:0.0:1788972116.254951:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17af400. 02000000:00000010:0.0:1788972116.254953:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b600. 00000100:00000010:0.0:1788972116.254955:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 02000000:00000010:0.0:1788972116.254958:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d17afc00. 02000000:00000010:0.0:1788972116.254960:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:0.0:1788972116.254962:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000100:00000010:3.0F:1788972117.266056:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467780. 00000100:00000010:2.0:1788972117.266452:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467ea0. 00000001:02000400:0.0:1788972117.996597:0:115509:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-lnet test 304: Check locked primary peer nid consolidation ========================================================== 12:41:56 (1788972116) 00000020:00000010:3.0:1788972118.322537:0:115533:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972118.322631:0:115533:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972118.365059:0:115538:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972118.365223:0:115538:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:1.0:1788972119.402358:0:115543:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972119.404076:0:115543:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:2.0:1788972120.439294:0:115547:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972120.439385:0:115547:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:0.0:1788972121.362189:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88380. 02000000:00000010:0.0:1788972121.362213:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cfff00. 00000100:00000010:0.0:1788972121.362253:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788972121.362258:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788972121.362265:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788972121.362269:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 02000000:00000010:0.0:1788972121.362348:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0187600. 00000400:00000200:0.0:1788972121.362358:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972121.362362:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972121.362368:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972121.362375:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586944, portal 10 00000400:00000010:0.0:1788972121.362380:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8000. 00000100:00000200:0.0:1788972121.362384:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189586944, offset 0 00000400:00000200:0.0:1788972121.362393:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972121.362401:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972121.362408:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972121.362413:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972121.362418:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972121.362421:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972121.362424:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972121.362428:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [576:576:256:4294967295] 192.168.203.150@tcp LPNI seq info [576:576:8:4294967295] 00000400:00000200:0.0:1788972121.362432:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972121.362442:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972121.362448:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972121.362454:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff200. 00000800:00000200:0.0:1788972121.362471:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972121.362478:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972121.362482:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972121.362498:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972121.362504:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c59c0. 00000100:00000010:0.0:1788972121.362527:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5cc0. 02000000:00000010:0.0:1788972121.362559:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0187800. 00000400:00000200:0.0:1788972121.362565:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972121.362567:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972121.362571:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972121.362574:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189586816, portal 25 00000400:00000010:0.0:1788972121.362577:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a85d8. 00000100:00000200:0.0:1788972121.362578:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189586816, offset 0 00000400:00000200:0.0:1788972121.362583:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972121.362597:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972121.362601:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972121.362604:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972121.362606:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972121.362609:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972121.362611:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972121.362614:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [577:577:255:4294967295] 192.168.203.150@tcp LPNI seq info [577:577:7:4294967295] 00000400:00000200:0.0:1788972121.362618:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972121.362628:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972121.362634:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972121.362638:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff800. 00000800:00000200:0.0:1788972121.362642:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972121.362646:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972121.362649:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972121.362653:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972121.362667:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0186200. 00000400:00000200:0.0:1788972121.362670:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972121.362672:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788972121.362676:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8110. 00000100:00000200:0.0:1788972121.362678:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587072, portal 10 00000400:00000010:0.0:1788972121.362683:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a87f8. 00000100:00000200:0.0:1788972121.362686:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189587072, offset 0 00000400:00000200:0.0:1788972121.362708:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000800:00000200:2.0:1788972121.362712:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1788972121.362713:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972121.362719:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972121.362722:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972121.362725:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972121.362727:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972121.362729:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972121.362732:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [578:578:254:4294967295] 192.168.203.150@tcp LPNI seq info [578:578:6:4294967295] 00000400:00000200:0.0:1788972121.362736:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972121.362742:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972121.362745:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972121.362749:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffc00. 00000800:00000200:0.0:1788972121.362753:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972121.362757:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972121.362759:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972121.362765:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:2.0:1788972121.364453:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff200. 00000400:00000200:2.0:1788972121.364465:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972121.364473:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972121.364480:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8000 00000400:00000010:2.0:1788972121.364482:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8000. 00000100:00000200:2.0:1788972121.364492:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a300 x1875872189586944/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972121.364536:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972121.364539:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff800. 00000400:00000200:2.0:1788972121.364541:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972121.364545:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972121.364547:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a85d8 00000400:00000010:2.0:1788972121.364549:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a85d8. 00000100:00000200:2.0:1788972121.364633:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88380 x1875872189586816/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972137 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972121.364690:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972121.364693:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffc00. 00000400:00000200:2.0:1788972121.364696:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972121.364700:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972121.364703:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a87f8 00000400:00000010:2.0:1788972121.364705:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a87f8. 00000100:00000200:2.0:1788972121.364711:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189587072/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972121.370705:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.370716:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972121.370736:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.370741:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.370746:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.370751:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811d80 00000400:00000200:2.0:1788972121.370768:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.370775:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c65 [1] + 224 00000400:00000200:2.0:1788972121.370791:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972121.370804:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972121.370846:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88380 x1875872189586816/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972121.371047:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972121.371053:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880 00000400:00000010:0.0:1788972121.371055:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972121.371062:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88380 x1875872189586816/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972121.371071:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88380 x1875872189586816/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972121.371097:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0187800. 02000000:00000010:0.0:1788972121.371103:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cfff00. 00000100:00000010:0.0:1788972121.371108:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88380. 00000400:00000200:2.0:1788972121.375148:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.375158:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972121.375167:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.375171:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.375176:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.375180:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811e80 00000400:00000200:2.0:1788972121.375185:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.375191:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c6d [1] + 224 00000400:00000200:2.0:1788972121.375206:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972121.375210:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972121.375222:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189587072/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972121.375325:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788972121.375331:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8110 00000400:00000010:0.0:1788972121.375333:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8110. 00000100:00000200:0.0:1788972121.375340:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189587072/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972121.375384:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189587072/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972121.375415:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0186200. 02000000:00000010:0.0:1788972121.375421:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:0.0:1788972121.375425:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 00000400:00000200:2.0:1788972121.377999:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.378008:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972121.378016:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.378020:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.378026:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.378029:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811e00 00000400:00000200:2.0:1788972121.378035:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972121.378042:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c5d [1] + 224 00000400:00000200:2.0:1788972121.378056:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972121.378061:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972121.378073:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a300 x1875872189586944/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972121.378236:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972121.378241:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088 00000400:00000010:0.0:1788972121.378243:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972121.378250:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a300 x1875872189586944/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972121.378259:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a300 x1875872189586944/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972137 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972121.378288:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0187600. 02000000:00000010:0.0:1788972121.378295:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788972121.378299:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a300. 00000020:00000010:1.0:1788972121.470191:0:115551:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972121.472095:0:115551:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000100:00000010:2.0:1788972122.387933:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c59c0. 00000100:00000010:3.0:1788972122.390221:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5cc0. 00000020:00000010:0.0:1788972122.497302:0:115555:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972122.497405:0:115555:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:3.0:1788972123.536464:0:115559:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972123.536553:0:115559:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:1.0:1788972124.576489:0:115564:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972124.576578:0:115564:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972125.616539:0:115568:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972125.616619:0:115568:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000100:00000010:1.0:1788972126.482391:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972126.482425:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bb00. 00000100:00000010:1.0:1788972126.482470:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972126.482480:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 00000100:00000010:1.0:1788972126.482488:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972126.482492:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 02000000:00000010:1.0:1788972126.482528:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c9542600. 00000400:00000200:1.0:1788972126.482539:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972126.482560:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972126.482568:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972126.482577:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587328, portal 10 00000400:00000010:1.0:1788972126.482585:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525908. 00000100:00000200:1.0:1788972126.482589:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189587328, offset 0 00000400:00000200:1.0:1788972126.482601:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972126.482610:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972126.482620:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972126.482627:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972126.482632:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972126.482637:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972126.482640:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972126.482644:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [579:579:256:4294967295] 192.168.203.150@tcp LPNI seq info [579:579:8:4294967295] 00000400:00000200:1.0:1788972126.482649:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972126.482660:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972126.483287:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972126.483295:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:1.0:1788972126.483303:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972126.483311:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972126.483317:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972126.483338:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972126.483344:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467780. 00000100:00000010:1.0:1788972126.483360:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467180. 02000000:00000010:1.0:1788972126.483392:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73c9543400. 00000400:00000200:1.0:1788972126.483397:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972126.483400:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972126.483405:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972126.483408:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587200, portal 25 00000400:00000010:1.0:1788972126.483412:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525990. 00000100:00000200:1.0:1788972126.483415:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189587200, offset 0 00000400:00000200:1.0:1788972126.483419:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972126.483432:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972126.483437:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972126.483440:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972126.483443:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972126.483446:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972126.483449:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972126.483452:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [580:580:255:4294967295] 192.168.203.150@tcp LPNI seq info [580:580:7:4294967295] 00000400:00000200:1.0:1788972126.483457:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972126.483463:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972126.483468:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972126.483471:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972126.483475:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972126.483479:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972126.483482:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972126.483486:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972126.483500:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600. 00000400:00000200:1.0:1788972126.483503:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972126.483505:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972126.483509:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972126.483511:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587456, portal 10 00000400:00000010:1.0:1788972126.483516:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75252a8. 00000100:00000200:1.0:1788972126.483518:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189587456, offset 0 00000400:00000200:1.0:1788972126.483522:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972126.483525:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972126.483528:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972126.483530:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972126.483532:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972126.483534:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972126.483537:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972126.483540:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [581:581:254:4294967295] 192.168.203.150@tcp LPNI seq info [581:581:6:4294967295] 00000400:00000200:1.0:1788972126.483544:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972126.483555:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972126.483559:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972126.483562:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b900. 00000800:00000200:1.0:1788972126.483566:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972126.483569:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972126.483572:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972126.483577:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972126.483881:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972126.483887:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:2.0:1788972126.483894:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972126.483900:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972126.483907:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525908 00000400:00000010:2.0:1788972126.483909:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525908. 00000100:00000200:2.0:1788972126.483921:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189587328/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972126.483945:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972126.483947:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:2.0:1788972126.483949:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972126.483952:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972126.483954:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525990 00000400:00000010:2.0:1788972126.483956:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525990. 00000100:00000200:2.0:1788972126.483960:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1c00 x1875872189587200/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972142 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972126.485135:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972126.485144:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b900. 00000400:00000200:2.0:1788972126.485150:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972126.485157:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972126.485163:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75252a8 00000400:00000010:2.0:1788972126.485166:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75252a8. 00000100:00000200:2.0:1788972126.485238:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189587456/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972126.489255:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489267:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972126.489276:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489281:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489288:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489307:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887811f80 00000400:00000200:2.0:1788972126.489314:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489322:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c75 [1] + 224 00000400:00000200:2.0:1788972126.489334:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972126.489339:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972126.489355:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189587328/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972126.489502:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489506:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972126.489510:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489513:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489515:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489519:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812000 00000400:00000200:2.0:1788972126.489522:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489525:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c85 [1] + 224 00000400:00000200:2.0:1788972126.489531:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972126.489534:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972126.489541:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189587456/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972126.489553:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489556:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972126.489560:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489562:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489565:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489568:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887811f00 00000400:00000200:2.0:1788972126.489571:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972126.489575:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c7d [1] + 224 00000400:00000200:2.0:1788972126.489583:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972126.489587:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972126.489600:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1c00 x1875872189587200/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972126.493874:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972126.493892:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:1.0:1788972126.493896:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972126.493914:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189587328/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972126.493927:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189587328/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972126.493976:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c9542600. 02000000:00000010:1.0:1788972126.493987:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:1.0:1788972126.493992:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 00000400:00000010:1.0:1788972126.494077:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972126.494081:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:1.0:1788972126.494083:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972126.494088:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1c00 x1875872189587200/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972126.494098:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1c00 x1875872189587200/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972126.494112:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972126.494114:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972126.494116:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972126.494121:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189587456/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972126.494160:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189587456/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972142 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972126.494244:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73c9543400. 02000000:00000010:1.0:1788972126.494249:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bb00. 00000100:00000010:1.0:1788972126.494252:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972126.494255:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600. 02000000:00000010:1.0:1788972126.494261:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:1.0:1788972126.494263:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000020:00000010:3.0:1788972126.658713:0:115572:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972126.661737:0:115572:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000100:00000010:2.0:1788972127.505993:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467780. 00000100:00000010:3.0:1788972127.506487:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467180. 00000020:00000010:3.0:1788972127.707516:0:115576:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972127.708431:0:115576:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:0.0:1788972128.745427:0:115580:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972128.745528:0:115580:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:2.0:1788972129.791313:0:115584:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972129.791437:0:115584:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000020:00000010:3.0:1788972130.844524:0:115588:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972130.846596:0:115588:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000100:00000010:0.0:1788972131.605196:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788972131.605230:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788972131.605284:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788972131.605290:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 00000100:00000010:0.0:1788972131.605300:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88380. 02000000:00000010:0.0:1788972131.605305:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cfff00. 02000000:00000010:0.0:1788972131.605346:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0186200. 00000400:00000200:0.0:1788972131.605359:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972131.605365:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972131.605375:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972131.605406:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587712, portal 10 00000400:00000010:0.0:1788972131.605416:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8110. 00000100:00000200:0.0:1788972131.605422:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189587712, offset 0 00000400:00000200:0.0:1788972131.605437:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972131.605450:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972131.605460:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972131.605469:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972131.605475:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972131.605481:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972131.605485:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972131.605491:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [582:582:256:4294967295] 192.168.203.150@tcp LPNI seq info [582:582:8:4294967295] 00000400:00000200:0.0:1788972131.605497:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972131.605510:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972131.605520:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972131.605527:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff300. 00000800:00000200:0.0:1788972131.605537:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972131.605549:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972131.605555:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972131.605587:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972131.605595:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5d20. 00000100:00000010:0.0:1788972131.605611:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5000. 02000000:00000010:0.0:1788972131.605636:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0187600. 00000400:00000200:0.0:1788972131.605641:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972131.605644:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788972131.605649:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972131.605652:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587584, portal 25 00000400:00000010:0.0:1788972131.605656:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8f68. 00000100:00000200:0.0:1788972131.605658:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189587584, offset 0 00000400:00000200:0.0:1788972131.605664:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972131.605668:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972131.605671:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972131.605681:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972131.605684:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972131.605687:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972131.605691:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972131.605694:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [583:583:255:4294967295] 192.168.203.150@tcp LPNI seq info [583:583:7:4294967295] 00000400:00000200:0.0:1788972131.605699:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972131.605706:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972131.605712:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972131.605716:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff600. 00000800:00000200:0.0:1788972131.605721:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972131.605739:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972131.605741:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972131.605746:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972131.605755:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 02000000:00000010:0.0:1788972131.605762:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0187800. 00000800:00000010:2.0:1788972131.605763:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff300. 00000400:00000200:0.0:1788972131.605770:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.605772:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1788972131.605774:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972131.605780:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000400:00000200:2.0:1788972131.605782:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:0.0:1788972131.605785:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587840, portal 10 00000400:00000200:2.0:1788972131.605791:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8110 00000400:00000010:0.0:1788972131.605792:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8550. 00000400:00000010:2.0:1788972131.605796:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8110. 00000100:00000200:2.0:1788972131.605808:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189587712/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972131.605865:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972131.605868:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff600. 00000400:00000200:2.0:1788972131.605871:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972131.605875:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:0.0:1788972131.605935:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189587840, offset 0 00000400:00000200:2.0:1788972131.605947:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8f68 00000400:00000010:2.0:1788972131.605951:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8f68. 00000100:00000200:2.0:1788972131.605959:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a300 x1875872189587584/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1788972131.607102:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972131.607116:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972131.607124:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972131.607130:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972131.607132:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972131.607136:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972131.607139:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972131.607143:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [584:584:256:4294967295] 192.168.203.150@tcp LPNI seq info [584:584:8:4294967295] 00000400:00000200:0.0:1788972131.607147:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972131.607157:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972131.607180:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972131.607188:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff600. 00000800:00000200:0.0:1788972131.607195:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972131.607202:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972131.607206:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972131.607235:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972131.608256:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972131.608263:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff600. 00000400:00000200:2.0:1788972131.608270:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972131.608276:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972131.608284:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8550 00000400:00000010:2.0:1788972131.608287:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8550. 00000100:00000200:2.0:1788972131.608298:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88380 x1875872189587840/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972131.609634:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.609646:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972131.609672:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.609676:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.609683:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.609686:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812100 00000400:00000200:2.0:1788972131.609692:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.609699:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c8d [1] + 224 00000400:00000200:2.0:1788972131.609716:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972131.609720:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972131.609746:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189587712/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972131.612141:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612155:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972131.612167:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612174:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612180:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612184:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812080 00000400:00000200:2.0:1788972131.612191:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612200:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c95 [1] + 224 00000400:00000200:2.0:1788972131.612213:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972131.612218:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972131.612235:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a300 x1875872189587584/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972131.612265:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612269:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972131.612274:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612276:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612279:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612282:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812180 00000400:00000200:2.0:1788972131.612285:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972131.612308:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1c9d [1] + 224 00000400:00000200:2.0:1788972131.612322:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972131.612325:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972131.612333:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88380 x1875872189587840/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972131.614153:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972131.614170:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088 00000400:00000010:0.0:1788972131.614174:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972131.614191:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189587712/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972131.614204:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189587712/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972131.614251:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0186200. 02000000:00000010:0.0:1788972131.614260:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:0.0:1788972131.614266:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 00000400:00000010:0.0:1788972131.614279:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788972131.614281:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880 00000400:00000010:0.0:1788972131.614283:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972131.614288:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a300 x1875872189587584/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972131.614298:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a300 x1875872189587584/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972131.614312:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972131.614378:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788972131.614380:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972131.614388:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88380 x1875872189587840/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972131.614397:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88380 x1875872189587840/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972147 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972131.614436:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0187600. 02000000:00000010:0.0:1788972131.614440:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788972131.614442:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788972131.614445:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0187800. 02000000:00000010:0.0:1788972131.614447:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cfff00. 00000100:00000010:0.0:1788972131.614449:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88380. 00000020:00000010:2.0:1788972131.878689:0:115593:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972131.878803:0:115593:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:2.0:1788972132.626165:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5d20. 00000100:00000010:3.0:1788972132.629361:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5000. 00000020:00000010:0.0:1788972132.922325:0:115597:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972132.922431:0:115597:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972133.972018:0:115601:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972133.972255:0:115601:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972135.028345:0:115605:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972135.028450:0:115605:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972136.081913:0:115609:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972136.082025:0:115609:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000100:00000010:0.0:1788972136.722317:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88380. 02000000:00000010:0.0:1788972136.722348:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cfff00. 00000100:00000010:0.0:1788972136.722396:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8a300. 02000000:00000010:0.0:1788972136.722402:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 00000100:00000010:0.0:1788972136.722410:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89f80. 02000000:00000010:0.0:1788972136.722413:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 02000000:00000010:1.0:1788972136.724107:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835800. 00000400:00000200:1.0:1788972136.724127:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972136.724133:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972136.724143:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972136.724165:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189587968, portal 25 00000400:00000010:1.0:1788972136.724172:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525880. 00000100:00000200:1.0:1788972136.724177:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189587968, offset 0 00000400:00000200:1.0:1788972136.724188:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972136.724200:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972136.724211:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972136.724217:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972136.724222:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972136.724227:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972136.724230:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972136.724234:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [585:585:256:4294967295] 192.168.203.150@tcp LPNI seq info [585:585:8:4294967295] 00000400:00000200:1.0:1788972136.724239:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972136.724250:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972136.724257:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972136.724263:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:1.0:1788972136.724270:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972136.724279:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972136.724301:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972136.724331:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972136.724337:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74678a0. 00000100:00000010:1.0:1788972136.724353:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467720. 02000000:00000010:1.0:1788972136.724368:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835c00. 00000400:00000200:1.0:1788972136.724372:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972136.724375:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972136.724379:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972136.724383:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588224, portal 10 00000400:00000010:1.0:1788972136.724388:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525000. 00000100:00000200:1.0:1788972136.724390:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189588224, offset 0 00000400:00000200:1.0:1788972136.724395:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972136.724399:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972136.724403:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972136.724406:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972136.724409:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972136.724412:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972136.724414:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972136.724417:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [586:586:255:4294967295] 192.168.203.150@tcp LPNI seq info [586:586:7:4294967295] 00000400:00000200:1.0:1788972136.724421:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972136.724428:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972136.724432:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972136.724435:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bb00. 00000800:00000200:1.0:1788972136.724439:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972136.724444:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972136.724447:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972136.724453:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972136.724474:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835000. 00000400:00000200:1.0:1788972136.724478:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972136.724480:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972136.724484:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525440. 00000100:00000200:1.0:1788972136.724492:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588096, portal 10 00000400:00000010:1.0:1788972136.724497:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525088. 00000100:00000200:1.0:1788972136.724500:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189588096, offset 0 00000400:00000200:1.0:1788972136.724504:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972136.724508:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972136.724512:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972136.724515:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972136.724517:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972136.724519:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972136.724522:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972136.724525:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [587:587:254:4294967295] 192.168.203.150@tcp LPNI seq info [587:587:6:4294967295] 00000400:00000200:1.0:1788972136.724529:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972136.724535:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972136.724540:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972136.724544:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b100. 00000800:00000200:1.0:1788972136.724548:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972136.724552:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972136.724555:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972136.724560:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972136.725151:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972136.725158:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:3.0:1788972136.725168:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972136.725175:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972136.725183:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525880 00000400:00000010:3.0:1788972136.725186:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525880. 00000100:00000200:3.0:1788972136.725197:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88380 x1875872189587968/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972136.725226:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972136.725229:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bb00. 00000400:00000200:3.0:1788972136.725231:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972136.725235:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972136.725237:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525000 00000400:00000010:3.0:1788972136.725239:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525000. 00000100:00000200:3.0:1788972136.725244:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89f80 x1875872189588224/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972136.725325:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972136.725335:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b100. 00000400:00000200:3.0:1788972136.725338:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972136.725342:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972136.725346:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525088 00000400:00000010:3.0:1788972136.725347:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525088. 00000100:00000200:3.0:1788972136.725353:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8a300 x1875872189588096/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972136.733291:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733304:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972136.733314:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733321:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733328:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733333:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812300 00000400:00000200:2.0:1788972136.733340:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733348:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1cad [1] + 224 00000400:00000200:2.0:1788972136.733363:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972136.733368:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972136.733387:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89f80 x1875872189588224/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972136.733430:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733434:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972136.733439:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733442:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733445:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733448:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812280 00000400:00000200:2.0:1788972136.733451:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972136.733456:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1cb5 [1] + 224 00000400:00000200:2.0:1788972136.733466:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972136.733487:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972136.733494:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8a300 x1875872189588096/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972136.733725:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972136.733731:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525440 00000400:00000010:1.0:1788972136.733735:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525440. 00000100:00000200:1.0:1788972136.733742:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8a300 x1875872189588096/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972136.733751:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8a300 x1875872189588096/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972136.733786:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972136.733789:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:1.0:1788972136.733791:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972136.733795:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89f80 x1875872189588224/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972136.733844:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89f80 x1875872189588224/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972136.733866:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835000. 02000000:00000010:1.0:1788972136.733876:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:1.0:1788972136.733881:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8a300. 02000000:00000010:1.0:1788972136.733886:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835c00. 02000000:00000010:1.0:1788972136.733888:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:1.0:1788972136.733891:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89f80. 00000400:00000200:3.0:1788972136.742601:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972136.742614:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972136.742625:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972136.742630:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972136.742637:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972136.742641:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812200 00000400:00000200:3.0:1788972136.742647:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972136.742655:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ca5 [1] + 224 00000400:00000200:3.0:1788972136.742672:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972136.742677:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972136.742709:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88380 x1875872189587968/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972136.742915:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972136.742922:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972136.742925:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972136.742938:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88380 x1875872189587968/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972136.742948:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88380 x1875872189587968/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972152 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972136.743051:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835800. 02000000:00000010:1.0:1788972136.743062:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cfff00. 00000100:00000010:1.0:1788972136.743068:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88380. 00000020:00000010:2.0:1788972137.127397:0:115613:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972137.127512:0:115613:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:2.0:1788972137.745960:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74678a0. 00000100:00000010:3.0:1788972137.745976:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467720. 00000020:00000010:1.0:1788972138.172897:0:115617:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972138.173032:0:115617:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:3.0:1788972139.225080:0:115622:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972139.225181:0:115622:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972140.269270:0:115626:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972140.269391:0:115626:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:2.0:1788972141.295585:0:115630:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972141.295700:0:115630:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:3.0:1788972141.842334:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790000. 02000000:00000010:3.0:1788972141.842368:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972141.842414:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8793480. 02000000:00000010:3.0:1788972141.842429:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 00000100:00000010:3.0:1788972141.842437:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792d80. 02000000:00000010:3.0:1788972141.842441:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 02000000:00000010:2.0:1788972141.842555:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624600. 00000400:00000200:2.0:1788972141.842567:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972141.842572:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00. 00000400:00000010:2.0:1788972141.842580:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972141.842590:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588352, portal 25 00000400:00000010:2.0:1788972141.842597:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bee0. 00000100:00000200:2.0:1788972141.842600:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189588352, offset 0 00000400:00000200:2.0:1788972141.842611:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972141.842621:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972141.842630:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972141.842636:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972141.842641:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972141.842646:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972141.842649:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972141.842665:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [588:588:256:4294967295] 192.168.203.150@tcp LPNI seq info [588:588:8:4294967295] 00000400:00000200:2.0:1788972141.842670:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972141.842681:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972141.842687:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972141.842694:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63500. 00000800:00000200:2.0:1788972141.842702:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972141.842710:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972141.842715:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972141.842744:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972141.842750:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e3c0. 00000100:00000010:2.0:1788972141.842763:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed80. 02000000:00000010:2.0:1788972141.842774:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624e00. 00000400:00000200:2.0:1788972141.842789:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972141.842792:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972141.842796:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972141.842799:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588608, portal 10 00000400:00000010:2.0:1788972141.842804:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b550. 00000100:00000200:2.0:1788972141.842807:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189588608, offset 0 00000400:00000200:2.0:1788972141.842812:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972141.842854:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972141.842858:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972141.842861:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972141.842863:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972141.842866:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972141.842868:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972141.842870:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [589:589:255:4294967295] 192.168.203.150@tcp LPNI seq info [589:589:7:4294967295] 00000400:00000200:2.0:1788972141.842874:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972141.842879:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972141.842883:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972141.842886:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972141.842890:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972141.842894:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972141.842896:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972141.842902:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972141.842922:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624c00. 00000400:00000200:2.0:1788972141.842925:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972141.842927:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126420. 00000400:00000010:2.0:1788972141.842930:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972141.842932:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588480, portal 10 00000400:00000010:2.0:1788972141.842943:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b6e8. 00000100:00000200:2.0:1788972141.842946:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189588480, offset 0 00000400:00000200:2.0:1788972141.842950:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972141.842953:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972141.842956:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972141.842958:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972141.842960:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972141.842963:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972141.842965:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972141.842967:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [590:590:254:4294967295] 192.168.203.150@tcp LPNI seq info [590:590:6:4294967295] 00000400:00000200:2.0:1788972141.842971:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972141.842976:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972141.842979:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972141.842982:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00. 00000800:00000200:2.0:1788972141.842985:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972141.842989:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972141.842992:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972141.842996:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972141.843408:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972141.843423:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63500. 00000400:00000200:3.0:1788972141.843433:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972141.843439:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972141.843447:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bee0 00000400:00000010:3.0:1788972141.843450:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bee0. 00000100:00000200:3.0:1788972141.843461:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790000 x1875872189588352/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972157 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972141.843488:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972141.843490:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:3.0:1788972141.843494:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972141.843497:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972141.843500:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b550 00000400:00000010:3.0:1788972141.843502:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b550. 00000100:00000200:3.0:1788972141.843506:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792d80 x1875872189588608/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972141.843591:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972141.843595:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00. 00000400:00000200:3.0:1788972141.843597:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972141.843600:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972141.843603:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b6e8 00000400:00000010:3.0:1788972141.843604:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b6e8. 00000100:00000200:3.0:1788972141.843609:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8793480 x1875872189588480/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972141.847933:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.847946:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972141.847956:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.847962:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.847968:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.847972:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812480 00000400:00000200:3.0:1788972141.847978:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848003:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ccd [1] + 224 00000400:00000200:3.0:1788972141.848016:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972141.848021:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972141.848037:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792d80 x1875872189588608/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972141.848079:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848083:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972141.848086:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848089:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848091:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848094:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812380 00000400:00000200:3.0:1788972141.848097:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848102:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1cbd [1] + 224 00000400:00000200:3.0:1788972141.848108:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972141.848111:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972141.848117:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790000 x1875872189588352/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972141.848131:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848134:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972141.848138:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848140:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848143:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848145:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812400 00000400:00000200:3.0:1788972141.848148:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972141.848152:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1cdd [1] + 224 00000400:00000200:3.0:1788972141.848160:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972141.848164:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972141.848170:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8793480 x1875872189588480/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972141.848495:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126420. 00000400:00000200:2.0:1788972141.848503:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972141.848506:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972141.848514:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8793480 x1875872189588480/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972141.848524:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8793480 x1875872189588480/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972141.848562:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00. 00000400:00000200:2.0:1788972141.848565:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68 00000400:00000010:2.0:1788972141.848567:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972141.848571:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790000 x1875872189588352/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972141.848590:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790000 x1875872189588352/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972141.848604:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:2.0:1788972141.848606:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972141.848608:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972141.848611:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792d80 x1875872189588608/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972141.848618:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792d80 x1875872189588608/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972157 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972141.848635:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624c00. 02000000:00000010:2.0:1788972141.848643:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788972141.848650:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8793480. 02000000:00000010:2.0:1788972141.848667:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624600. 02000000:00000010:2.0:1788972141.848670:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788972141.848673:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790000. 02000000:00000010:2.0:1788972141.848675:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624e00. 02000000:00000010:2.0:1788972141.848677:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788972141.848679:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792d80. 00000020:00000010:1.0:1788972142.345414:0:115634:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972142.346275:0:115634:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000100:00000010:2.0:1788972142.866965:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e3c0. 00000100:00000010:3.0:1788972142.869309:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed80. 00000020:00000010:1.0:1788972143.373796:0:115638:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972143.373947:0:115638:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:0.0:1788972144.399862:0:115642:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972144.400123:0:115642:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:2.0:1788972145.438648:0:115646:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972145.439183:0:115646:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000020:00000010:0.0:1788972146.484234:0:115651:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972146.484388:0:115651:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000100:00000010:1.0:1788972146.962316:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3480. 02000000:00000010:1.0:1788972146.962425:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b000. 00000100:00000010:1.0:1788972146.962467:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f1c00. 02000000:00000010:1.0:1788972146.962473:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b300. 00000100:00000010:1.0:1788972146.962480:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3100. 02000000:00000010:1.0:1788972146.962508:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b100. 02000000:00000010:0.0:1788972146.962521:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0186600. 00000400:00000200:0.0:1788972146.962535:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972146.962541:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972146.962549:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972146.962559:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588736, portal 25 00000400:00000010:0.0:1788972146.962565:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8880. 00000100:00000200:0.0:1788972146.962576:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189588736, offset 0 00000400:00000200:0.0:1788972146.962589:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972146.962599:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972146.962609:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972146.963032:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972146.963039:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972146.963048:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972146.963052:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972146.963057:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [591:591:256:4294967295] 192.168.203.150@tcp LPNI seq info [591:591:8:4294967295] 00000400:00000200:0.0:1788972146.963063:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972146.963075:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972146.963082:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972146.963090:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff300. 00000800:00000200:0.0:1788972146.963098:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972146.963107:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972146.963113:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972146.963135:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972146.963142:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c56c0. 00000100:00000010:0.0:1788972146.963158:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5840. 02000000:00000010:0.0:1788972146.963185:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0186200. 00000400:00000200:0.0:1788972146.963189:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972146.963193:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788972146.963197:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972146.963201:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588992, portal 10 00000400:00000010:0.0:1788972146.963207:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8330. 00000100:00000200:0.0:1788972146.963210:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189588992, offset 0 00000400:00000200:0.0:1788972146.963214:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972146.963219:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972146.963223:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972146.963226:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972146.963228:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972146.963237:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972146.963240:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972146.963243:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [592:592:255:4294967295] 192.168.203.150@tcp LPNI seq info [592:592:7:4294967295] 00000400:00000200:0.0:1788972146.963247:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972146.963254:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972146.963259:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972146.963262:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffc00. 00000800:00000200:0.0:1788972146.963266:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972146.963271:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972146.963273:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972146.963279:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:0.0:1788972146.963294:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0187600. 00000400:00000200:0.0:1788972146.963299:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972146.963302:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972146.963306:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8aa0. 00000100:00000200:0.0:1788972146.963310:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189588864, portal 10 00000400:00000010:0.0:1788972146.963315:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8b28. 00000100:00000200:0.0:1788972146.963317:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189588864, offset 0 00000400:00000200:0.0:1788972146.963322:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972146.963325:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972146.963329:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972146.963332:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972146.963335:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972146.963337:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972146.963339:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972146.963342:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [593:593:254:4294967295] 192.168.203.150@tcp LPNI seq info [593:593:6:4294967295] 00000400:00000200:0.0:1788972146.963346:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972146.963351:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972146.963356:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972146.963359:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff800. 00000800:00000200:0.0:1788972146.963369:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972146.963374:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972146.963376:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972146.963382:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972146.963458:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972146.963464:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff300. 00000400:00000200:3.0:1788972146.963473:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972146.963479:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972146.963486:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8880 00000400:00000010:3.0:1788972146.963489:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8880. 00000100:00000200:3.0:1788972146.963500:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3480 x1875872189588736/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972146.963539:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972146.963659:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffc00. 00000400:00000200:3.0:1788972146.963665:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972146.963669:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972146.963672:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8330 00000400:00000010:3.0:1788972146.963674:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8330. 00000100:00000200:3.0:1788972146.963680:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3100 x1875872189588992/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972146.963766:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972146.963769:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff800. 00000400:00000200:3.0:1788972146.963772:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972146.963776:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972146.963779:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8b28 00000400:00000010:3.0:1788972146.963780:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8b28. 00000100:00000200:3.0:1788972146.963785:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f1c00 x1875872189588864/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972146.970977:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.970987:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972146.970997:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971002:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971008:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971026:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812500 00000400:00000200:3.0:1788972146.971032:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971038:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ced [1] + 224 00000400:00000200:3.0:1788972146.971050:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972146.971055:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972146.971068:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3480 x1875872189588736/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972146.971104:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971108:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972146.971112:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971115:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971123:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971126:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812600 00000400:00000200:3.0:1788972146.971129:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971133:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1cf5 [1] + 224 00000400:00000200:3.0:1788972146.971140:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972146.971143:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972146.971152:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3100 x1875872189588992/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972146.971167:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971170:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972146.971174:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971176:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971179:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971182:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812580 00000400:00000200:3.0:1788972146.971185:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972146.971189:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1cfd [1] + 224 00000400:00000200:3.0:1788972146.971197:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972146.971200:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972146.971213:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f1c00 x1875872189588864/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972146.971353:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972146.971366:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8aa0 00000400:00000010:0.0:1788972146.971369:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8aa0. 00000100:00000200:0.0:1788972146.971383:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f1c00 x1875872189588864/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972146.971395:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f1c00 x1875872189588864/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972146.971438:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:0.0:1788972146.971440:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088 00000400:00000010:0.0:1788972146.971443:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972146.971448:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3100 x1875872189588992/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972146.971457:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3100 x1875872189588992/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972146.971482:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972146.971485:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788972146.971488:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972146.971493:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3480 x1875872189588736/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972146.971501:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3480 x1875872189588736/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972162 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972146.971518:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0187600. 02000000:00000010:0.0:1788972146.971526:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b300. 00000100:00000010:0.0:1788972146.971532:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f1c00. 02000000:00000010:0.0:1788972146.971536:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0186200. 02000000:00000010:0.0:1788972146.971554:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b100. 00000100:00000010:0.0:1788972146.971556:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3100. 02000000:00000010:0.0:1788972146.971560:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0186600. 02000000:00000010:0.0:1788972146.971562:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b000. 00000100:00000010:0.0:1788972146.971564:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3480. 00000020:00000010:2.0:1788972147.521620:0:115655:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972147.521715:0:115655:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:2.0:1788972147.986112:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c56c0. 00000100:00000010:3.0:1788972147.986216:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5840. 00000020:00000010:0.0:1788972148.566550:0:115659:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972148.566661:0:115659:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972149.605607:0:115663:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972149.605713:0:115663:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972150.642491:0:115667:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972150.642578:0:115667:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:3.0:1788972151.681090:0:115671:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972151.681201:0:115671:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000100:00000010:2.0:1788972152.082276:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f800. 02000000:00000010:2.0:1788972152.082301:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972152.082344:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788972152.082349:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63300. 00000100:00000010:2.0:1788972152.082356:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e300. 02000000:00000010:2.0:1788972152.082387:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 02000000:00000010:2.0:1788972152.082417:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624000. 00000400:00000200:2.0:1788972152.082427:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972152.082431:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972152.082437:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972152.082443:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589248, portal 10 00000400:00000010:2.0:1788972152.082450:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972152.082452:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189589248, offset 0 00000400:00000200:2.0:1788972152.082461:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972152.082469:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.082477:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972152.082482:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972152.082486:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972152.082490:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972152.082494:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972152.082498:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [594:594:256:4294967295] 192.168.203.150@tcp LPNI seq info [594:594:8:4294967295] 00000400:00000200:2.0:1788972152.082503:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.082512:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 02000000:00000010:3.0:1788972152.082514:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e000. 00000800:00000200:2.0:1788972152.082521:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972152.082525:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000010:2.0:1788972152.082529:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000400:00000010:3.0:1788972152.082531:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972152.082538:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000800:00000200:2.0:1788972152.082539:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000100:00000200:3.0:1788972152.082542:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589120, portal 25 00000400:00000010:3.0:1788972152.082548:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788972152.082550:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189589120, offset 0 00000400:00000200:3.0:1788972152.082556:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000800:00000200:2.0:1788972152.082560:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:3.0:1788972152.082561:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972152.082568:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972152.082570:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972152.082576:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972152.082588:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972152.082591:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972152.082594:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972152.082597:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.082602:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [595:595:255:4294967295] 192.168.203.150@tcp LPNI seq info [595:595:7:4294967295] 00000100:00000010:2.0:1788972152.082605:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e3c0. 00000400:00000200:3.0:1788972152.082609:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.082618:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000100:00000010:2.0:1788972152.082620:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed20. 00000800:00000200:3.0:1788972152.082625:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972152.082630:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972152.082635:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972152.082641:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972152.082644:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972152.082650:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972152.082664:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fc00. 00000400:00000200:3.0:1788972152.082668:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972152.082676:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000010:3.0:1788972152.082678:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000800:00000010:2.0:1788972152.082681:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000010:3.0:1788972152.082684:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972152.082688:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589376, portal 10 00000400:00000200:2.0:1788972152.082689:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1788972152.082695:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000400:00000200:2.0:1788972152.082697:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972152.082700:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189589376, offset 0 00000400:00000200:2.0:1788972152.082703:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68 00000400:00000010:2.0:1788972152.082714:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68. 00000400:00000200:3.0:1788972152.082722:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000100:00000200:2.0:1788972152.082725:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189589248/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972152.082731:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.082739:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972152.082743:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972152.082746:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972152.082748:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972152.082749:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972152.082752:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [596:596:254:4294967295] 192.168.203.150@tcp LPNI seq info [596:596:6:4294967295] 00000400:00000200:3.0:1788972152.082755:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.082760:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972152.082764:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972152.082768:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972152.082772:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972152.082777:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972152.082780:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972152.082785:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972152.082847:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972152.082850:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972152.082854:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972152.082858:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972152.082860:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:2.0:1788972152.082862:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:2.0:1788972152.082867:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f800 x1875872189589120/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972152.082883:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972152.082885:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972152.082888:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972152.082890:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972152.082893:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:2.0:1788972152.082894:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:2.0:1788972152.082898:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e300 x1875872189589376/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972152.085218:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.085228:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972152.085236:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.085240:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.085244:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.085248:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812700 00000400:00000200:3.0:1788972152.085253:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972152.085258:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d05 [1] + 224 00000400:00000200:3.0:1788972152.085283:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972152.085288:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972152.085311:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189589248/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972152.086272:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:2.0:1788972152.086280:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972152.086283:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972152.086292:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189589248/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972152.086303:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189589248/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972152.086336:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624000. 02000000:00000010:2.0:1788972152.086343:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63300. 00000100:00000010:2.0:1788972152.086347:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000400:00000200:2.0:1788972152.098382:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098407:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972152.098416:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098421:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098426:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098430:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812680 00000400:00000200:2.0:1788972152.098436:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098442:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d15 [1] + 224 00000400:00000200:2.0:1788972152.098452:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972152.098457:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972152.098470:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f800 x1875872189589120/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972152.098513:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098517:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972152.098521:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098523:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098526:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098529:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812780 00000400:00000200:2.0:1788972152.098532:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972152.098536:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d25 [1] + 224 00000400:00000200:2.0:1788972152.098547:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972152.098550:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972152.098557:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e300 x1875872189589376/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972152.098679:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972152.098684:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972152.098686:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972152.098692:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f800 x1875872189589120/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972152.098701:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f800 x1875872189589120/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972152.098722:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:3.0:1788972152.098724:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:3.0:1788972152.098725:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972152.098729:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e300 x1875872189589376/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972152.098736:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e300 x1875872189589376/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972168 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972152.098753:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e000. 02000000:00000010:3.0:1788972152.098758:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:3.0:1788972152.098762:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f800. 02000000:00000010:3.0:1788972152.098765:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fc00. 02000000:00000010:3.0:1788972152.098767:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:3.0:1788972152.098769:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e300. 00000020:00000010:0.0:1788972152.727331:0:115675:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972152.727722:0:115675:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000100:00000010:2.0:1788972153.105968:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e3c0. 00000100:00000010:3.0:1788972153.106499:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed20. 00000020:00000010:1.0:1788972153.783006:0:115680:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972153.783226:0:115680:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:3.0:1788972154.825182:0:115684:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972154.825276:0:115684:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972155.877487:0:115688:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972155.877587:0:115688:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:2.0:1788972156.902092:0:115692:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972156.902193:0:115692:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:1.0:1788972157.203264:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f2d80. 02000000:00000010:1.0:1788972157.203297:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1bb00. 02000000:00000010:0.0:1788972157.203418:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0186600. 00000400:00000200:0.0:1788972157.203431:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972157.203437:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972157.203446:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972157.203455:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589504, portal 25 00000400:00000010:0.0:1788972157.203462:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8088. 00000100:00000200:0.0:1788972157.203466:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189589504, offset 0 00000400:00000200:0.0:1788972157.203787:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972157.203801:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972157.203840:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972157.203847:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972157.203852:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972157.203857:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972157.203860:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972157.203864:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [597:597:256:4294967295] 192.168.203.150@tcp LPNI seq info [597:597:8:4294967295] 00000400:00000200:0.0:1788972157.203870:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972157.203881:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972157.203888:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972157.203913:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff200. 00000800:00000200:0.0:1788972157.203921:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972157.203930:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972157.203936:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972157.204020:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972157.204027:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5960. 00000100:00000010:0.0:1788972157.204044:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5ae0. 00000100:00000010:1.0:1788972157.204109:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f0700. 02000000:00000010:1.0:1788972157.204119:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b500. 00000800:00000200:3.0:1788972157.204365:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972157.204371:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff200. 00000400:00000200:3.0:1788972157.204378:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972157.204390:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972157.204398:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8088 00000400:00000010:3.0:1788972157.204401:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8088. 00000100:00000200:3.0:1788972157.204411:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f2d80 x1875872189589504/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972157.207977:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d0186400. 00000400:00000200:0.0:1788972157.207999:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972157.208005:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff43c8. 00000400:00000010:0.0:1788972157.208014:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8aa0. 00000100:00000200:0.0:1788972157.208023:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589632, portal 10 00000400:00000010:0.0:1788972157.208033:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8c38. 00000100:00000200:0.0:1788972157.208038:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189589632, offset 0 00000400:00000200:0.0:1788972157.208051:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972157.208063:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972157.208291:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972157.208299:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972157.208304:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972157.208310:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972157.208395:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972157.208457:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [598:598:256:4294967295] 192.168.203.150@tcp LPNI seq info [598:598:8:4294967295] 00000400:00000200:0.0:1788972157.208467:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972157.208479:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972157.208486:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972157.208494:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffe00. 00000800:00000200:0.0:1788972157.208501:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972157.208509:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972157.208516:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffe00 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:1.0:1788972157.208892:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c87f3800. 02000000:00000010:1.0:1788972157.208913:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c2e1b900. 02000000:00000010:1.0:1788972157.209012:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835400. 00000400:00000200:1.0:1788972157.209019:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972157.209023:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972157.209028:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972157.209033:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589760, portal 10 00000400:00000010:1.0:1788972157.209040:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525770. 00000100:00000200:1.0:1788972157.209043:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189589760, offset 0 00000400:00000200:1.0:1788972157.209050:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972157.209170:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972157.209176:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972157.209180:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972157.209184:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972157.209187:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972157.209189:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972157.209192:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [599:599:255:4294967295] 192.168.203.150@tcp LPNI seq info [599:599:7:4294967295] 00000400:00000200:1.0:1788972157.209196:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972157.209203:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972157.209208:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972157.209212:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972157.209216:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972157.209222:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972157.209226:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972157.209465:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.209489:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.209495:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972157.209502:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.209506:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.209511:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.209515:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812800 00000400:00000200:3.0:1788972157.209521:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.209528:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d35 [1] + 224 00000400:00000200:3.0:1788972157.209653:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972157.209659:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972157.209675:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f2d80 x1875872189589504/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972157.209811:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:1.0:1788972157.210599:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972157.210708:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788972157.210711:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972157.210724:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f2d80 x1875872189589504/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972157.210736:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f2d80 x1875872189589504/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972157.210772:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0186600. 02000000:00000010:0.0:1788972157.210781:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1bb00. 00000100:00000010:0.0:1788972157.210787:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f2d80. 00000800:00000200:3.0:1788972157.212528:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972157.212552:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffe00. 00000400:00000200:3.0:1788972157.212562:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972157.212572:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972157.212582:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8c38 00000400:00000010:3.0:1788972157.212586:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8c38. 00000100:00000200:3.0:1788972157.212599:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f0700 x1875872189589632/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972157.212851:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972157.212855:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:3.0:1788972157.212859:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972157.212864:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972157.212868:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525770 00000400:00000010:3.0:1788972157.212870:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525770. 00000100:00000200:3.0:1788972157.212895:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c87f3800 x1875872189589760/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972157.214415:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972157.214426:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972157.214435:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972157.214454:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972157.214461:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972157.214465:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812880 00000400:00000200:2.0:1788972157.214481:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972157.214488:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d3d [1] + 224 00000400:00000200:2.0:1788972157.214507:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972157.214511:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972157.214527:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f0700 x1875872189589632/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972157.214941:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff43c8. 00000400:00000200:1.0:1788972157.214949:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8aa0 00000400:00000010:1.0:1788972157.214952:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8aa0. 00000100:00000200:1.0:1788972157.214959:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f0700 x1875872189589632/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972157.214969:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f0700 x1875872189589632/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972157.215011:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d0186400. 02000000:00000010:1.0:1788972157.215019:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b500. 00000100:00000010:1.0:1788972157.215024:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f0700. 00000400:00000200:3.0:1788972157.217245:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.217256:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972157.217266:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.217272:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.217278:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.217282:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812900 00000400:00000200:3.0:1788972157.217289:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972157.217296:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d45 [1] + 224 00000400:00000200:3.0:1788972157.217370:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972157.217376:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972157.217392:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c87f3800 x1875872189589760/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972157.217989:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972157.217998:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972157.218001:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972157.218010:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c87f3800 x1875872189589760/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972157.218021:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c87f3800 x1875872189589760/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972173 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972157.218060:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835400. 02000000:00000010:1.0:1788972157.218070:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c2e1b900. 00000100:00000010:1.0:1788972157.218075:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c87f3800. 00000020:00000010:0.0:1788972157.955371:0:115696:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972157.955483:0:115696:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000100:00000010:3.0:1788972158.226369:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5ae0. 00000100:00000010:2.0:1788972158.227685:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5960. 00000020:00000010:1.0:1788972158.988004:0:115700:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972158.988228:0:115700:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972160.018381:0:115704:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972160.018482:0:115704:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:0.0:1788972161.057483:0:115709:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972161.057608:0:115709:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972162.085031:0:115713:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972162.085135:0:115713:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000100:00000010:3.0:1788972162.323130:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791f80. 02000000:00000010:3.0:1788972162.323197:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 02000000:00000010:2.0:1788972162.323322:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624000. 00000400:00000200:2.0:1788972162.323334:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972162.323340:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972162.323349:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972162.323358:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189589888, portal 25 00000400:00000010:2.0:1788972162.323365:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bf68. 00000100:00000200:2.0:1788972162.323368:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189589888, offset 0 00000400:00000200:2.0:1788972162.323381:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972162.323390:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972162.323400:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972162.323406:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972162.323410:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972162.323415:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972162.323419:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972162.323423:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [600:600:256:4294967295] 192.168.203.150@tcp LPNI seq info [600:600:8:4294967295] 00000400:00000200:2.0:1788972162.323428:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972162.323439:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972162.323447:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972162.323453:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63300. 00000800:00000200:2.0:1788972162.323461:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972162.323490:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972162.323496:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63300 type 1, nob 320 niov 1 nkiov 1 00000100:00000010:3.0:1788972162.323533:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790380. 02000000:00000010:3.0:1788972162.323540:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 00000100:00000010:3.0:1788972162.323556:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792680. 02000000:00000010:3.0:1788972162.323563:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 02000000:00000010:3.0:1788972162.323589:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788972162.323593:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972162.323596:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436160. 00000400:00000010:3.0:1788972162.323600:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972162.323602:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590016, portal 10 00000400:00000010:3.0:1788972162.323607:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972162.323609:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189590016, offset 0 00000400:00000200:3.0:1788972162.323614:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972162.323618:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.323627:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972162.323630:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972162.323632:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972162.323634:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972162.323636:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972162.323639:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [601:601:255:4294967295] 192.168.203.150@tcp LPNI seq info [601:601:7:4294967295] 00000400:00000200:3.0:1788972162.323643:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.323648:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972162.323652:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972162.323656:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972162.323659:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972162.323674:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972162.323676:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972162.324159:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.324165:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972162.324167:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e3c0. 00000100:00000010:2.0:1788972162.324180:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ea80. 02000000:00000010:2.0:1788972162.324203:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625c00. 00000400:00000200:2.0:1788972162.324207:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972162.324210:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00. 00000400:00000010:2.0:1788972162.324214:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972162.324218:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590144, portal 10 00000400:00000010:2.0:1788972162.324223:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b990. 00000100:00000200:2.0:1788972162.324230:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189590144, offset 0 00000400:00000200:2.0:1788972162.324235:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972162.324239:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972162.324243:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972162.324246:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972162.324248:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972162.324250:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972162.324252:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972162.324255:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [602:602:254:4294967295] 192.168.203.150@tcp LPNI seq info [602:602:6:4294967295] 00000400:00000200:2.0:1788972162.324259:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972162.324264:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:2.0:1788972162.324265:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000010:3.0:1788972162.324268:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63300. 00000800:00000200:2.0:1788972162.324270:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972162.324273:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000800:00000200:2.0:1788972162.324277:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972162.324282:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972162.324285:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972162.324286:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972162.324289:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.324291:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972162.324298:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bf68 00000400:00000010:3.0:1788972162.324301:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bf68. 00000100:00000200:3.0:1788972162.324312:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791f80 x1875872189589888/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972178 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972162.324337:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972162.324340:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:3.0:1788972162.324343:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972162.324346:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972162.324349:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972162.324350:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972162.324355:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790380 x1875872189590016/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972162.324450:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972162.324461:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000200:3.0:1788972162.324464:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972162.324468:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972162.324471:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b990 00000400:00000010:3.0:1788972162.324473:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b990. 00000100:00000200:3.0:1788972162.324479:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792680 x1875872189590144/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972162.329195:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329205:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972162.329213:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329218:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329223:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329227:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812a00 00000400:00000200:3.0:1788972162.329233:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329239:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d5d [1] + 224 00000400:00000200:3.0:1788972162.329250:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972162.329255:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972162.329269:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790380 x1875872189590016/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972162.329311:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329314:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972162.329318:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329321:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329324:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329327:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812a80 00000400:00000200:3.0:1788972162.329331:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329336:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d6d [1] + 224 00000400:00000200:3.0:1788972162.329343:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972162.329349:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972162.329355:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792680 x1875872189590144/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972162.329394:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436160. 00000400:00000200:2.0:1788972162.329407:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:2.0:1788972162.329410:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:2.0:1788972162.329435:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790380 x1875872189590016/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972162.329449:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:2.0:1788972162.329462:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790380 x1875872189590016/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972162.329470:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972162.329483:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329487:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329493:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329497:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812980 00000400:00000200:3.0:1788972162.329502:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972162.329523:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d4d [1] + 224 02000000:00000010:2.0:1788972162.329526:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:2.0:1788972162.329538:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788972162.329556:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790380. 00000400:00000200:3.0:1788972162.329573:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972162.329579:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972162.329587:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791f80 x1875872189589888/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972162.329620:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00. 00000400:00000200:2.0:1788972162.329627:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:2.0:1788972162.329629:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972162.329640:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792680 x1875872189590144/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972162.329650:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792680 x1875872189590144/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972162.329668:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:2.0:1788972162.329671:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972162.329673:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972162.329678:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791f80 x1875872189589888/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972162.329687:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791f80 x1875872189589888/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972178 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972162.329712:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625c00. 02000000:00000010:2.0:1788972162.329718:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788972162.329722:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792680. 02000000:00000010:2.0:1788972162.329725:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624000. 02000000:00000010:2.0:1788972162.329729:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788972162.329731:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791f80. 00000020:00000010:1.0:1788972163.117733:0:115717:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972163.118006:0:115717:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000100:00000010:3.0:1788972163.345973:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ea80. 00000100:00000010:2.0:1788972163.346198:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e3c0. 00000020:00000010:1.0:1788972164.152988:0:115721:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972164.153071:0:115721:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:0.0:1788972165.199404:0:115725:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972165.199498:0:115725:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:3.0:1788972166.235361:0:115729:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972166.235459:0:115729:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972167.263347:0:115733:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972167.263433:0:115733:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000100:00000010:2.0:1788972167.443033:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c000. 02000000:00000010:2.0:1788972167.443062:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63700. 00000100:00000010:2.0:1788972167.443097:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972167.443103:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63900. 00000100:00000010:2.0:1788972167.443109:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972167.443113:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63300. 02000000:00000010:2.0:1788972167.443141:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625c00. 00000400:00000200:2.0:1788972167.443151:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972167.443155:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972167.443161:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972167.443168:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590400, portal 10 00000400:00000010:2.0:1788972167.443175:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bcc0. 00000100:00000200:2.0:1788972167.443177:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189590400, offset 0 00000400:00000200:2.0:1788972167.443187:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972167.443195:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972167.443202:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972167.443208:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972167.443212:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972167.443216:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972167.443219:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972167.443223:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [603:603:256:4294967295] 192.168.203.150@tcp LPNI seq info [603:603:8:4294967295] 00000400:00000200:2.0:1788972167.443228:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972167.443237:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972167.443243:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972167.443248:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63c00. 00000800:00000200:2.0:1788972167.443255:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972167.443263:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972167.443278:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972167.443292:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972167.443297:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972167.443308:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e5a0. 02000000:00000010:2.0:1788972167.443332:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624c00. 00000400:00000200:2.0:1788972167.443342:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972167.443344:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126b00. 00000400:00000010:2.0:1788972167.443347:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b880. 00000100:00000200:2.0:1788972167.443350:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590272, portal 25 00000400:00000010:2.0:1788972167.443354:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bb28. 00000100:00000200:2.0:1788972167.443356:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189590272, offset 0 00000400:00000200:2.0:1788972167.443361:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972167.443365:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972167.443368:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972167.443372:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972167.443374:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972167.443377:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972167.443379:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972167.443381:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [604:604:255:4294967295] 192.168.203.150@tcp LPNI seq info [604:604:7:4294967295] 00000400:00000200:2.0:1788972167.443395:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972167.443401:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972167.443405:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972167.443409:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63d00. 00000800:00000200:2.0:1788972167.443413:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972167.443417:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972167.443420:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63d00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972167.443425:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972167.443436:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624000. 00000400:00000200:2.0:1788972167.443440:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972167.443442:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126420. 00000400:00000010:2.0:1788972167.443444:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972167.443447:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590528, portal 10 00000400:00000010:2.0:1788972167.443451:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b2a8. 00000100:00000200:2.0:1788972167.443453:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189590528, offset 0 00000400:00000200:2.0:1788972167.443458:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972167.443461:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972167.443488:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972167.443490:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972167.443493:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972167.443495:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972167.443497:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972167.443500:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [605:605:254:4294967295] 192.168.203.150@tcp LPNI seq info [605:605:6:4294967295] 00000400:00000200:2.0:1788972167.443504:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972167.443509:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972167.443513:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972167.443517:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00. 00000800:00000200:2.0:1788972167.443520:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972167.443524:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972167.443526:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972167.443531:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972167.444154:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972167.444160:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63c00. 00000400:00000200:3.0:1788972167.444165:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972167.444171:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972167.444177:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bcc0 00000400:00000010:3.0:1788972167.444179:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bcc0. 00000100:00000200:3.0:1788972167.444188:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189590400/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972167.444209:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972167.444212:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63d00. 00000400:00000200:3.0:1788972167.444214:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972167.444217:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972167.444227:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bb28 00000400:00000010:3.0:1788972167.444229:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bb28. 00000100:00000200:3.0:1788972167.444233:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c000 x1875872189590272/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972183 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972167.444291:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972167.444294:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00. 00000400:00000200:3.0:1788972167.444297:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972167.444300:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972167.444302:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b2a8 00000400:00000010:3.0:1788972167.444304:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b2a8. 00000100:00000200:3.0:1788972167.444309:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189590528/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972167.447169:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.447180:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972167.447188:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.447192:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.447197:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.447201:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812b80 00000400:00000200:3.0:1788972167.447205:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.447211:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d7d [1] + 224 00000400:00000200:3.0:1788972167.447222:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972167.447227:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972167.447240:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189590400/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972167.448008:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:2.0:1788972167.448015:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972167.448017:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972167.448025:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189590400/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972167.448035:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189590400/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972167.448068:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625c00. 02000000:00000010:2.0:1788972167.448077:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63900. 00000100:00000010:2.0:1788972167.448081:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 00000400:00000200:3.0:1788972167.451184:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451196:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972167.451208:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451214:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451220:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451225:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812c00 00000400:00000200:3.0:1788972167.451299:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451307:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d9d [1] + 224 00000400:00000200:3.0:1788972167.451323:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972167.451329:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972167.451348:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189590528/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972167.451395:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451399:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972167.451403:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451406:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451409:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451411:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812b00 00000400:00000200:3.0:1788972167.451415:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972167.451419:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1d8d [1] + 224 00000400:00000200:3.0:1788972167.452231:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:2.0:1788972167.452453:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126420. 00000400:00000200:2.0:1788972167.452465:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:2.0:1788972167.452468:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972167.452491:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189590528/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972167.452503:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189590528/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972167.452559:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624000. 02000000:00000010:2.0:1788972167.452567:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63300. 00000100:00000010:2.0:1788972167.452573:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000400:00000200:3.0:1788972167.452847:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972167.452858:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c000 x1875872189590272/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972167.452901:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126b00. 00000400:00000200:3.0:1788972167.452907:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b880 00000400:00000010:3.0:1788972167.452910:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b880. 00000100:00000200:3.0:1788972167.452915:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c000 x1875872189590272/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972167.452925:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c000 x1875872189590272/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972183 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972167.453796:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624c00. 02000000:00000010:3.0:1788972167.453809:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63700. 00000100:00000010:3.0:1788972167.453856:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c000. 00000020:00000010:0.0:1788972168.315897:0:115738:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972168.315991:0:115738:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000100:00000010:2.0:1788972168.466006:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0. 00000100:00000010:3.0:1788972168.466203:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e5a0. 00000020:00000010:2.0:1788972169.370709:0:115742:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972169.370914:0:115742:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000020:00000010:1.0:1788972170.416522:0:115746:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972170.416670:0:115746:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:0.0:1788972171.478695:0:115750:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972171.478804:0:115750:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:2.0:1788972172.521248:0:115754:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972172.521425:0:115754:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:2.0:1788972172.561989:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972172.562022:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63300. 00000100:00000010:2.0:1788972172.562073:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ed80. 02000000:00000010:2.0:1788972172.562078:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63900. 00000100:00000010:2.0:1788972172.562085:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840ea00. 02000000:00000010:2.0:1788972172.562089:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 02000000:00000010:2.0:1788972172.562131:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625400. 00000400:00000200:2.0:1788972172.562142:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972172.562147:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126420. 00000400:00000010:2.0:1788972172.562156:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7be58. 00000100:00000200:2.0:1788972172.562165:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590784, portal 10 00000400:00000010:2.0:1788972172.562174:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972172.562178:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189590784, offset 0 00000400:00000200:2.0:1788972172.562189:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972172.562200:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.562209:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972172.562216:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972172.562220:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972172.562225:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972172.562229:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972172.562233:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [606:606:256:4294967295] 192.168.203.150@tcp LPNI seq info [606:606:8:4294967295] 00000400:00000200:2.0:1788972172.562238:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.562268:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972172.562277:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972172.562283:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63500. 00000800:00000200:2.0:1788972172.562291:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972172.562301:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972172.562307:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972172.562327:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972172.562334:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972172.562349:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e5a0. 02000000:00000010:2.0:1788972172.562375:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624000. 00000400:00000200:2.0:1788972172.562379:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972172.562382:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972172.562386:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b330. 00000100:00000200:2.0:1788972172.562390:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590656, portal 25 00000400:00000010:2.0:1788972172.562394:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788972172.562396:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189590656, offset 0 00000400:00000200:2.0:1788972172.562401:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972172.562405:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.562409:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972172.562412:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972172.562426:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972172.562429:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972172.562431:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972172.562442:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [607:607:255:4294967295] 192.168.203.150@tcp LPNI seq info [607:607:7:4294967295] 00000400:00000200:2.0:1788972172.562447:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.562452:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972172.562457:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972172.562461:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63200. 00000800:00000200:2.0:1788972172.562465:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972172.562469:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972172.562477:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972172.562482:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972172.562496:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624e00. 00000400:00000200:2.0:1788972172.562499:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972172.562502:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126478. 00000400:00000010:2.0:1788972172.562505:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bdd0. 00000100:00000200:2.0:1788972172.562508:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189590912, portal 10 00000400:00000010:2.0:1788972172.562512:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972172.562514:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189590912, offset 0 00000400:00000200:2.0:1788972172.562519:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972172.562522:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.562526:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972172.562529:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972172.562532:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972172.562533:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972172.562536:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972172.562538:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [608:608:254:4294967295] 192.168.203.150@tcp LPNI seq info [608:608:6:4294967295] 00000400:00000200:2.0:1788972172.562542:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.562548:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972172.562553:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972172.562556:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63100. 00000800:00000200:2.0:1788972172.562560:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972172.562564:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972172.562566:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972172.562571:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972172.562629:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972172.562634:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63500. 00000400:00000200:2.0:1788972172.562640:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972172.562645:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972172.562652:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972172.562655:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972172.562673:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ed80 x1875872189590784/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972172.562702:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972172.562705:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63200. 00000400:00000200:2.0:1788972172.562708:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972172.562711:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972172.562714:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b5d8 00000400:00000010:2.0:1788972172.562716:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788972172.562720:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189590656/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972172.562805:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972172.562808:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63100. 00000400:00000200:2.0:1788972172.562811:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972172.562856:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972172.562859:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972172.562861:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972172.562866:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840ea00 x1875872189590912/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972172.565651:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972172.565661:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972172.565671:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972172.565676:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972172.565682:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972172.565686:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812c80 00000400:00000200:3.0:1788972172.565692:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972172.565698:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1dbd [1] + 224 00000400:00000200:3.0:1788972172.565722:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972172.565728:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972172.565743:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189590656/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972172.567155:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:2.0:1788972172.567172:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b330 00000400:00000010:2.0:1788972172.567176:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b330. 00000100:00000200:2.0:1788972172.567194:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189590656/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972172.567208:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189590656/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972172.567245:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624000. 02000000:00000010:2.0:1788972172.567256:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63300. 00000100:00000010:2.0:1788972172.567261:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 00000400:00000200:2.0:1788972172.567583:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567589:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972172.567601:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567626:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567632:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567637:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812d00 00000400:00000200:2.0:1788972172.567643:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567651:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1dad [1] + 224 00000400:00000200:2.0:1788972172.567664:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972172.567669:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972172.567680:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ed80 x1875872189590784/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972172.567726:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567731:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972172.567735:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567738:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567740:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567743:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812d80 00000400:00000200:2.0:1788972172.567746:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972172.567751:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1dcd [1] + 224 00000400:00000200:2.0:1788972172.567871:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972172.567876:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972172.567883:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840ea00 x1875872189590912/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972172.567970:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126478. 00000400:00000200:3.0:1788972172.567975:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bdd0 00000400:00000010:3.0:1788972172.567978:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bdd0. 00000100:00000200:3.0:1788972172.567984:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ea00 x1875872189590912/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972172.567992:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ea00 x1875872189590912/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972172.568019:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126420. 00000400:00000200:3.0:1788972172.568022:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7be58 00000400:00000010:3.0:1788972172.568024:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7be58. 00000100:00000200:3.0:1788972172.568029:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840ed80 x1875872189590784/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972172.568037:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840ed80 x1875872189590784/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972188 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972172.568053:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624e00. 02000000:00000010:3.0:1788972172.568058:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:3.0:1788972172.568061:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ea00. 02000000:00000010:3.0:1788972172.568074:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625400. 02000000:00000010:3.0:1788972172.568079:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63900. 00000100:00000010:3.0:1788972172.568082:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840ed80. 00000020:00000010:2.0:1788972173.572408:0:115758:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972173.575741:0:115758:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:3.0:1788972173.585886:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e5a0. 00000100:00000010:2.0:1788972173.587245:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0. 00000020:00000010:3.0:1788972174.623564:0:115763:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972174.623649:0:115763:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:0.0:1788972175.669049:0:115767:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73d4780890. 00000020:00000010:0.0:1788972175.669137:0:115767:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73d4780890. 00000020:00000010:2.0:1788972176.724804:0:115771:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972176.724910:0:115771:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:3.0:1788972177.682553:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788972177.682575:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972177.682603:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790700. 02000000:00000010:3.0:1788972177.682609:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972177.682616:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791180. 02000000:00000010:3.0:1788972177.682620:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 02000000:00000010:3.0:1788972177.682646:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e400. 00000400:00000200:3.0:1788972177.682656:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972177.682660:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972177.682666:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972177.682673:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591040, portal 25 00000400:00000010:3.0:1788972177.682677:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788972177.682679:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189591040, offset 0 00000400:00000200:3.0:1788972177.682688:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972177.682696:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972177.682703:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972177.682710:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972177.682714:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972177.682718:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972177.682721:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972177.682724:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [609:609:256:4294967295] 192.168.203.150@tcp LPNI seq info [609:609:8:4294967295] 00000400:00000200:3.0:1788972177.682729:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972177.682739:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972177.682745:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972177.682750:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972177.682769:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972177.682777:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972177.682781:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972177.682794:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972177.682799:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09c60. 00000100:00000010:3.0:1788972177.682809:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09a20. 02000000:00000010:2.0:1788972177.682927:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625000. 00000400:00000200:2.0:1788972177.682933:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972177.682936:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c60. 00000400:00000010:2.0:1788972177.682940:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b330. 00000100:00000200:2.0:1788972177.682949:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591168, portal 10 00000400:00000010:2.0:1788972177.682953:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972177.682955:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189591168, offset 0 00000400:00000200:2.0:1788972177.682960:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972177.682964:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.682968:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972177.682970:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972177.682973:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972177.682975:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972177.682977:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972177.682996:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [610:610:255:4294967295] 192.168.203.150@tcp LPNI seq info [610:610:7:4294967295] 00000400:00000200:2.0:1788972177.683001:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.683007:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972177.683012:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972177.683017:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63300. 00000800:00000200:2.0:1788972177.683020:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972177.683025:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972177.683028:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972177.683034:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972177.683090:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972177.683094:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972177.683100:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972177.683104:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972177.683109:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8 00000400:00000010:2.0:1788972177.683111:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8. 00000100:00000200:2.0:1788972177.683120:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189591040/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972177.683190:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972177.683193:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63300. 00000400:00000200:2.0:1788972177.683196:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972177.683199:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972177.683207:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972177.683209:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972177.683214:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790700 x1875872189591168/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972177.683990:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ea00. 00000400:00000200:3.0:1788972177.683997:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972177.684001:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972177.684006:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972177.684010:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591296, portal 10 00000400:00000010:3.0:1788972177.684015:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788972177.684017:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189591296, offset 0 00000400:00000200:3.0:1788972177.684023:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972177.684028:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972177.684032:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972177.684035:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972177.684037:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972177.684039:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972177.684041:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972177.684044:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [611:611:256:4294967295] 192.168.203.150@tcp LPNI seq info [611:611:8:4294967295] 00000400:00000200:3.0:1788972177.684047:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972177.684053:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972177.684058:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972177.684062:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972177.684067:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972177.684072:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972177.684076:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972177.684098:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972177.684922:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972177.684928:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972177.684934:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972177.684939:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972177.684944:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873198 00000400:00000010:2.0:1788972177.684946:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873198. 00000100:00000200:2.0:1788972177.684953:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791180 x1875872189591296/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972177.687998:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688007:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972177.688015:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688020:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688026:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688029:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812e00 00000400:00000200:2.0:1788972177.688034:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688041:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ddd [1] + 224 00000400:00000200:2.0:1788972177.688053:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972177.688058:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972177.688071:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189591040/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972177.688152:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972177.688157:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972177.688168:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972177.688174:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189591040/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972177.688184:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189591040/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972177.688210:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e400. 02000000:00000010:3.0:1788972177.688215:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972177.688219:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 00000400:00000200:2.0:1788972177.688844:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688849:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972177.688854:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688857:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688861:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688864:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812e80 00000400:00000200:2.0:1788972177.688873:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688878:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ded [1] + 224 00000400:00000200:2.0:1788972177.688886:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972177.688889:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972177.688897:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790700 x1875872189591168/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972177.688926:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688929:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972177.688933:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688936:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688939:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688942:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887812f00 00000400:00000200:2.0:1788972177.688945:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972177.688949:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1dfd [1] + 224 00000400:00000200:2.0:1788972177.688994:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972177.688998:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972177.689004:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791180 x1875872189591296/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972177.689068:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c60. 00000400:00000200:3.0:1788972177.689072:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b330 00000400:00000010:3.0:1788972177.689074:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b330. 00000100:00000200:3.0:1788972177.689080:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790700 x1875872189591168/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972177.689089:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790700 x1875872189591168/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972177.689109:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625000. 02000000:00000010:3.0:1788972177.689114:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972177.689117:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790700. 00000400:00000010:3.0:1788972177.689132:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972177.689134:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:3.0:1788972177.689136:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972177.689141:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791180 x1875872189591296/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972177.689149:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791180 x1875872189591296/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972193 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972177.689160:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ea00. 02000000:00000010:3.0:1788972177.689162:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788972177.689164:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791180. 00000020:00000010:2.0:1788972177.772460:0:115775:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972177.772539:0:115775:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000100:00000010:3.0:1788972178.705961:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09a20. 00000100:00000010:2.0:1788972178.705965:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09c60. 00000020:00000010:1.0:1788972178.810933:0:115779:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972178.811022:0:115779:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000020:00000010:2.0:1788972179.868396:0:115783:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c6f4ed30. 00000020:00000010:2.0:1788972179.868490:0:115783:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c6f4ed30. 00000020:00000010:3.0:1788972180.906801:0:115787:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c69c5a10. 00000020:00000010:3.0:1788972180.906906:0:115787:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c69c5a10. 00000020:00000010:1.0:1788972180.928735:0:115788:0:(kernelcomm.c:98:lustre_device_list_start()) kmalloced '(glist)': 16 at ffff9a73c53a08d0. 00000020:00000010:1.0:1788972180.929021:0:115788:0:(kernelcomm.c:264:lustre_device_done()) kfreed 'glist': 16 at ffff9a73c53a08d0. 00000100:00000010:0.0:1788972182.802530:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88700. 02000000:00000010:0.0:1788972182.802630:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff500. 00000100:00000010:0.0:1788972182.802681:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b100. 02000000:00000010:0.0:1788972182.802688:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff000. 00000100:00000010:0.0:1788972182.802695:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972182.802699:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff400. 02000000:00000010:1.0:1788972182.802875:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600. 00000400:00000200:1.0:1788972182.802888:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972182.802893:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972182.802901:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972182.802910:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591424, portal 25 00000400:00000010:1.0:1788972182.802917:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525440. 00000100:00000200:1.0:1788972182.802921:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189591424, offset 0 00000400:00000200:1.0:1788972182.802932:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972182.802942:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972182.802951:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972182.803495:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972182.803504:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972182.803509:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972182.803512:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972182.803517:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [612:612:256:4294967295] 192.168.203.150@tcp LPNI seq info [612:612:8:4294967295] 00000400:00000200:1.0:1788972182.803524:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972182.803537:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972182.803544:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972182.803941:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b900. 00000800:00000200:1.0:1788972182.805073:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972182.805098:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972182.805105:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972182.805139:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972182.805151:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c74672a0. 00000100:00000010:1.0:1788972182.805167:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467a80. 02000000:00000010:1.0:1788972182.805197:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1302200. 00000400:00000200:1.0:1788972182.805223:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972182.805228:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972182.805236:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788972182.805245:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591680, portal 10 00000400:00000010:1.0:1788972182.805252:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525e58. 00000100:00000200:1.0:1788972182.805257:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189591680, offset 0 00000400:00000200:1.0:1788972182.805263:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972182.805272:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972182.805280:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972182.805286:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972182.805289:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972182.805293:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972182.805296:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972182.805299:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [613:613:255:4294967295] 192.168.203.150@tcp LPNI seq info [613:613:7:4294967295] 00000400:00000200:1.0:1788972182.805304:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972182.805313:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972182.805319:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972182.805325:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:1.0:1788972182.805330:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972182.805334:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972182.805337:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972182.805343:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972182.805382:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d40d8e00. 00000400:00000200:1.0:1788972182.805386:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972182.805390:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972182.805394:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788972182.805397:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591552, portal 10 00000400:00000010:1.0:1788972182.805402:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525f68. 00000100:00000200:1.0:1788972182.805404:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189591552, offset 0 00000400:00000200:1.0:1788972182.805409:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972182.805413:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972182.805421:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972182.805423:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972182.805425:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972182.805427:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972182.805428:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972182.805430:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [614:614:254:4294967295] 192.168.203.150@tcp LPNI seq info [614:614:6:4294967295] 00000400:00000200:1.0:1788972182.805433:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972182.805437:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972182.805440:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972182.805450:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:1.0:1788972182.805453:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972182.805456:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972182.805458:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972182.805462:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972182.806228:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972182.806237:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b900. 00000400:00000200:2.0:1788972182.806248:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972182.806255:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972182.806264:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525440 00000400:00000010:2.0:1788972182.806266:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525440. 00000100:00000200:2.0:1788972182.806280:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88700 x1875872189591424/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972182.806318:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972182.806321:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:2.0:1788972182.806323:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972182.806327:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972182.806330:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525e58 00000400:00000010:2.0:1788972182.806332:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525e58. 00000100:00000200:2.0:1788972182.806336:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189591680/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972182.806433:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972182.806436:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:2.0:1788972182.806440:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972182.806443:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972182.806455:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525f68 00000400:00000010:2.0:1788972182.806457:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525f68. 00000100:00000200:2.0:1788972182.806462:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b100 x1875872189591552/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972182.818258:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.818272:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972182.818283:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.818289:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.818296:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.818301:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887812f80 00000400:00000200:2.0:1788972182.818308:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.818315:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e0d [1] + 224 00000400:00000200:2.0:1788972182.818334:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972182.818340:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972182.818368:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88700 x1875872189591424/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972182.820827:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972182.820845:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972182.820848:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972182.820865:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88700 x1875872189591424/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972182.820879:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88700 x1875872189591424/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972182.821034:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600. 02000000:00000010:1.0:1788972182.821049:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff500. 00000100:00000010:1.0:1788972182.821054:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88700. 00000400:00000200:2.0:1788972182.831411:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.831448:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972182.831459:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.831466:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.831472:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.831477:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813080 00000400:00000200:2.0:1788972182.831484:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.831491:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e15 [1] + 224 00000400:00000200:2.0:1788972182.832132:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972182.832142:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972182.832162:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189591680/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972182.832211:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.832214:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972182.832219:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.832222:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.832225:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.832229:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813000 00000400:00000200:2.0:1788972182.832233:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972182.832238:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e1d [1] + 224 00000400:00000200:2.0:1788972182.832316:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972182.832320:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972182.832328:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b100 x1875872189591552/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972182.833408:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972182.833418:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:1.0:1788972182.833421:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788972182.833432:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b100 x1875872189591552/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972182.833442:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b100 x1875872189591552/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972182.833497:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972182.833685:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:1.0:1788972182.833687:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788972182.833695:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189591680/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972182.833704:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189591680/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972198 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972182.833726:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d40d8e00. 02000000:00000010:1.0:1788972182.833734:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff000. 00000100:00000010:1.0:1788972182.833739:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b100. 02000000:00000010:1.0:1788972182.833743:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1302200. 02000000:00000010:1.0:1788972182.833746:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff400. 00000100:00000010:1.0:1788972182.833748:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 00000100:00000010:3.0:1788972183.825968:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467a80. 00000100:00000010:2.0:1788972183.825972:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c74672a0. 00000100:00000010:2.0:1788972187.923893:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840fb80. 02000000:00000010:2.0:1788972187.923916:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63300. 00000100:00000010:2.0:1788972187.923951:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840c380. 02000000:00000010:2.0:1788972187.923956:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63100. 00000100:00000010:2.0:1788972187.923962:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840f100. 02000000:00000010:2.0:1788972187.923980:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63200. 02000000:00000010:3.0:1788972187.924104:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f800. 00000400:00000200:3.0:1788972187.924123:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972187.924129:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972187.924138:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972187.924148:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591808, portal 25 00000400:00000010:3.0:1788972187.924155:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972187.924159:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189591808, offset 0 00000400:00000200:3.0:1788972187.924171:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972187.924183:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.924193:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972187.924200:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972187.924205:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972187.924209:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972187.924212:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972187.924216:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [615:615:256:4294967295] 192.168.203.150@tcp LPNI seq info [615:615:8:4294967295] 00000400:00000200:3.0:1788972187.924221:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.924235:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972187.924245:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972187.924252:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972187.924261:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972187.924271:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972187.924276:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972187.924297:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972187.924303:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09a20. 00000100:00000010:3.0:1788972187.924317:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09de0. 02000000:00000010:3.0:1788972187.924340:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fa00. 00000400:00000200:3.0:1788972187.924344:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972187.924347:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972187.924363:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972187.924366:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592064, portal 10 00000400:00000010:3.0:1788972187.924372:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788972187.924374:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189592064, offset 0 00000400:00000200:3.0:1788972187.924379:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972187.924383:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.924386:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972187.924389:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972187.924391:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972187.924394:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972187.924396:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972187.924399:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [616:616:255:4294967295] 192.168.203.150@tcp LPNI seq info [616:616:7:4294967295] 00000400:00000200:3.0:1788972187.924403:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.924409:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972187.924413:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972187.924417:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67c00. 00000800:00000200:3.0:1788972187.924421:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972187.924425:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972187.924427:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972187.924434:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972187.924455:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ea00. 00000400:00000200:3.0:1788972187.924458:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972187.924460:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688. 00000400:00000010:3.0:1788972187.924463:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48732a8. 00000100:00000200:3.0:1788972187.924465:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189591936, portal 10 00000400:00000010:3.0:1788972187.924469:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873088. 00000100:00000200:3.0:1788972187.924471:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189591936, offset 0 00000400:00000200:3.0:1788972187.924476:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972187.924479:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.924493:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972187.924496:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972187.924504:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972187.924506:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972187.924509:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972187.924511:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [617:617:254:4294967295] 192.168.203.150@tcp LPNI seq info [617:617:6:4294967295] 00000400:00000200:3.0:1788972187.924515:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.924520:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972187.924524:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972187.924527:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972187.924531:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972187.924534:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972187.924537:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972187.924542:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972187.924600:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972187.924615:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:3.0:1788972187.924622:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972187.924627:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972187.924634:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972187.924636:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972187.924649:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840fb80 x1875872189591808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972203 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972187.924675:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972187.924677:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67c00. 00000400:00000200:3.0:1788972187.924680:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972187.924683:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972187.924686:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0 00000400:00000010:3.0:1788972187.924687:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788972187.924692:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840f100 x1875872189592064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972187.924772:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972187.924775:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:3.0:1788972187.924778:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972187.924787:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972187.924790:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873088 00000400:00000010:3.0:1788972187.924792:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873088. 00000100:00000200:3.0:1788972187.924797:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840c380 x1875872189591936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972187.932544:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932557:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972187.932569:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932575:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932582:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932587:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813100 00000400:00000200:2.0:1788972187.932603:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932616:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e25 [1] + 224 00000400:00000200:2.0:1788972187.932632:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972187.932637:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972187.932658:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840fb80 x1875872189591808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972187.932699:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932703:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972187.932707:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932710:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932712:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932715:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813200 00000400:00000200:2.0:1788972187.932718:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972187.932724:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e35 [1] + 224 00000400:00000200:2.0:1788972187.932746:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972187.932750:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972187.932757:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840f100 x1875872189592064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972187.932991:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972187.932998:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:3.0:1788972187.933001:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972187.933008:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840fb80 x1875872189591808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972187.933018:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840fb80 x1875872189591808/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972187.933047:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:3.0:1788972187.933050:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000010:3.0:1788972187.933052:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972187.933057:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840f100 x1875872189592064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972187.933065:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840f100 x1875872189592064/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972187.933091:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f800. 02000000:00000010:3.0:1788972187.933099:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63300. 00000100:00000010:3.0:1788972187.933105:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840fb80. 02000000:00000010:3.0:1788972187.933110:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fa00. 02000000:00000010:3.0:1788972187.933112:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63200. 00000100:00000010:3.0:1788972187.933114:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840f100. 00000400:00000200:3.0:1788972187.933453:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.933459:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972187.933465:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.933469:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.933472:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.933494:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813180 00000400:00000200:3.0:1788972187.933499:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972187.933504:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e45 [1] + 224 00000400:00000200:3.0:1788972187.933514:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972187.933518:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972187.933527:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840c380 x1875872189591936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972187.933561:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688. 00000400:00000200:3.0:1788972187.933564:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48732a8 00000400:00000010:3.0:1788972187.933566:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48732a8. 00000100:00000200:3.0:1788972187.933571:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840c380 x1875872189591936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972187.933580:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840c380 x1875872189591936/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972203 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972187.933610:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ea00. 02000000:00000010:3.0:1788972187.933614:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63100. 00000100:00000010:3.0:1788972187.933617:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840c380. 00000100:00000010:2.0:1788972188.947472:0:94543:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09a20. 00000100:00000010:3.0:1788972188.947580:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09de0. 00000100:00000010:3.0:1788972193.041956:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791180. 02000000:00000010:3.0:1788972193.041988:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 00000100:00000010:3.0:1788972193.042038:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790700. 02000000:00000010:3.0:1788972193.042044:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972193.042052:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788972193.042056:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 02000000:00000010:2.0:1788972193.042089:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 02000000:00000010:3.0:1788972193.042099:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:2.0:1788972193.042104:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.042109:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972193.042111:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c08. 00000400:00000010:3.0:1788972193.042114:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436688. 00000400:00000010:3.0:1788972193.042122:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48732a8. 00000400:00000010:2.0:1788972193.042124:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:3.0:1788972193.042130:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592320, portal 10 00000400:00000010:3.0:1788972193.042139:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:2.0:1788972193.042147:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592192, portal 25 00000100:00000200:3.0:1788972193.042154:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189592320, offset 0 00000400:00000010:2.0:1788972193.042155:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b5d8. 00000100:00000200:2.0:1788972193.042161:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189592192, offset 0 00000400:00000200:3.0:1788972193.042170:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972193.042179:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972193.042183:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.042194:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972193.042200:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972193.042205:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972193.042210:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972193.042212:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972193.042216:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [618:618:256:4294967295] 192.168.203.150@tcp LPNI seq info [618:618:8:4294967295] 00000400:00000200:3.0:1788972193.042221:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.042231:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000400:00000200:2.0:1788972193.042240:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972193.042241:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972193.042249:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000800:00000010:3.0:1788972193.042251:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000400:00000200:2.0:1788972193.042253:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972193.042257:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972193.042260:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000800:00000200:3.0:1788972193.042265:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:2.0:1788972193.042266:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972193.042273:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [619:619:255:4294967295] 192.168.203.150@tcp LPNI seq info [619:619:7:4294967295] 00000400:00000200:2.0:1788972193.042278:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972193.042283:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972193.042302:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972193.042320:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972193.042327:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09de0. 00000100:00000010:3.0:1788972193.042340:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09000. 00000800:00000200:3.0:1788972193.042483:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972193.042487:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000200:3.0:1788972193.042492:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972193.042497:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972193.043499:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972193.043510:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972193.043520:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63500. 00000800:00000200:2.0:1788972193.043527:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972193.043534:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972193.043539:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972193.043549:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972193.043577:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd000. 00000400:00000200:2.0:1788972193.043583:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972193.043587:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972193.043594:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972193.043600:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592448, portal 10 00000400:00000010:2.0:1788972193.043607:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972193.043610:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189592448, offset 0 00000400:00000200:2.0:1788972193.043615:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972193.044340:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000200:2.0:1788972193.044343:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972193.044346:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000400:00000200:2.0:1788972193.044355:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972193.044362:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000200:3.0:1788972193.044364:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790700 x1875872189592320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972193.044368:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972193.044392:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972193.044395:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972193.044399:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [620:620:254:4294967295] 192.168.203.150@tcp LPNI seq info [620:620:6:4294967295] 00000400:00000200:2.0:1788972193.044404:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972193.044413:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972193.044419:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972193.044425:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63100. 00000800:00000200:2.0:1788972193.044431:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972193.044437:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972193.044440:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972193.044449:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972193.044461:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972193.044467:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63500. 00000400:00000200:3.0:1788972193.044472:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972193.044478:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972193.044493:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b5d8 00000400:00000010:3.0:1788972193.044495:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b5d8. 00000100:00000200:3.0:1788972193.044502:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791180 x1875872189592192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972193.044583:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972193.044586:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63100. 00000400:00000200:3.0:1788972193.044589:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972193.044593:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972193.044595:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:3.0:1788972193.044597:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:3.0:1788972193.044602:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189592448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972193.051276:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.051298:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972193.051311:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.051317:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.051324:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.051328:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813300 00000400:00000200:3.0:1788972193.051335:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.051342:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e55 [1] + 224 00000400:00000200:3.0:1788972193.051360:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972193.051365:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972193.051385:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790700 x1875872189592320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972193.051725:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436688. 00000400:00000200:2.0:1788972193.051735:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48732a8 00000400:00000010:2.0:1788972193.051749:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48732a8. 00000100:00000200:2.0:1788972193.051758:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790700 x1875872189592320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972193.051768:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790700 x1875872189592320/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972193.051948:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:2.0:1788972193.051956:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788972193.051962:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790700. 00000400:00000200:3.0:1788972193.054938:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.054951:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972193.054962:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.054968:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.054974:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.054979:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813380 00000400:00000200:3.0:1788972193.055005:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.055012:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e75 [1] + 224 00000400:00000200:3.0:1788972193.055027:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972193.055032:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972193.055048:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189592448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972193.055085:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.055089:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972193.055093:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.055095:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.055098:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.055101:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813280 00000400:00000200:3.0:1788972193.055104:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972193.055108:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e5d [1] + 224 00000400:00000200:3.0:1788972193.055118:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972193.055122:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972193.055129:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791180 x1875872189592192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972193.055207:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c08. 00000400:00000200:2.0:1788972193.055214:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972193.055218:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972193.055227:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791180 x1875872189592192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972193.055239:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791180 x1875872189592192/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972193.055265:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972193.055274:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972193.055276:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972193.055281:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189592448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972193.057206:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189592448/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972209 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972193.057252:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:2.0:1788972193.057262:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788972193.057269:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791180. 02000000:00000010:2.0:1788972193.057274:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd000. 02000000:00000010:2.0:1788972193.057277:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788972193.057279:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 00000100:00000010:2.0:1788972194.065963:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09de0. 00000100:00000010:3.0:1788972194.066188:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09000. 00000100:00000010:3.0:1788972198.162214:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788972198.162246:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 00000100:00000010:3.0:1788972198.162305:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791880. 02000000:00000010:3.0:1788972198.162313:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788972198.162322:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788972198.162326:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 02000000:00000010:2.0:1788972198.162558:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd000. 00000400:00000200:2.0:1788972198.162571:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972198.162577:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972198.162585:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972198.162604:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592576, portal 25 00000400:00000010:2.0:1788972198.162612:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972198.162616:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189592576, offset 0 00000400:00000200:2.0:1788972198.162628:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972198.162639:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.162650:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972198.162657:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972198.162663:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972198.162667:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972198.162670:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972198.162675:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [621:621:256:4294967295] 192.168.203.150@tcp LPNI seq info [621:621:8:4294967295] 00000400:00000200:2.0:1788972198.162679:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.162691:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972198.162699:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972198.162727:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63200. 00000800:00000200:2.0:1788972198.162736:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972198.162746:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972198.162752:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63200 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972198.162775:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972198.162781:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972198.162795:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e3c0. 02000000:00000010:2.0:1788972198.162808:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 00000400:00000200:2.0:1788972198.162851:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972198.162855:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c08. 00000400:00000010:2.0:1788972198.162860:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972198.162864:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592832, portal 10 00000400:00000010:2.0:1788972198.162870:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972198.162874:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189592832, offset 0 00000400:00000200:2.0:1788972198.162880:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972198.162884:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.162888:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972198.162890:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972198.162893:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972198.162895:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972198.162897:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972198.162901:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [622:622:255:4294967295] 192.168.203.150@tcp LPNI seq info [622:622:7:4294967295] 00000400:00000200:2.0:1788972198.162905:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.162911:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972198.162916:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972198.162920:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63300. 00000800:00000200:2.0:1788972198.162924:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972198.162929:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972198.162932:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972198.162938:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972198.162951:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972198.162957:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63200. 00000400:00000200:3.0:1788972198.162965:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:2.0:1788972198.162969:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dc000. 00000400:00000200:3.0:1788972198.162973:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972198.162978:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.162981:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972198.162982:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:3.0:1788972198.162993:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:3.0:1788972198.163005:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790e00 x1875872189592576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972198.163036:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972198.163039:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63300. 00000400:00000200:3.0:1788972198.163042:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972198.163045:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000010:2.0:1788972198.163332:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972198.163339:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592704, portal 10 00000400:00000010:2.0:1788972198.163345:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788972198.163347:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189592704, offset 0 00000400:00000200:2.0:1788972198.163353:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972198.163358:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.163362:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972198.163365:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972198.163367:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972198.163370:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972198.163373:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972198.163376:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [623:623:255:4294967295] 192.168.203.150@tcp LPNI seq info [623:623:7:4294967295] 00000400:00000200:2.0:1788972198.163381:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.163387:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972198.163391:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972198.163395:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000800:00000200:2.0:1788972198.163399:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972198.163404:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972198.163407:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972198.163412:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.166663:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:3.0:1788972198.166675:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:3.0:1788972198.166699:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790a80 x1875872189592832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972198.166765:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.166771:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972198.166779:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.166783:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.166789:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.166793:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813400 00000400:00000200:3.0:1788972198.166799:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.166836:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e85 [1] + 224 00000400:00000200:3.0:1788972198.166864:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972198.166869:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972198.166879:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790e00 x1875872189592576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972198.166935:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972198.166939:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000200:3.0:1788972198.166944:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972198.166948:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972198.166951:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:3.0:1788972198.166953:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:3.0:1788972198.166959:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791880 x1875872189592704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972198.172574:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972198.172593:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972198.172596:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972198.172614:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790e00 x1875872189592576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972198.172629:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790e00 x1875872189592576/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972198.172670:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd000. 02000000:00000010:2.0:1788972198.172702:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788972198.172709:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790e00. 00000400:00000200:2.0:1788972198.179511:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.179525:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972198.179549:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.179555:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.179562:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.179567:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813500 00000400:00000200:2.0:1788972198.179585:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972198.179592:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1e95 [1] + 224 00000400:00000200:2.0:1788972198.179615:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972198.179621:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972198.179639:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790a80 x1875872189592832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972198.179719:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c08. 00000400:00000200:2.0:1788972198.179725:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:2.0:1788972198.179728:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972198.179734:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790a80 x1875872189592832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972198.179744:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790a80 x1875872189592832/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972198.179783:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:2.0:1788972198.179791:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788972198.179798:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790a80. 00000400:00000200:3.0:1788972198.182233:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.182246:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972198.182269:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.182275:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.182281:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.182286:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813480 00000400:00000200:3.0:1788972198.182293:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972198.182300:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ea5 [1] + 224 00000400:00000200:3.0:1788972198.182317:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972198.182322:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972198.182338:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791880 x1875872189592704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972198.182672:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:2.0:1788972198.182679:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972198.182682:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972198.182692:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791880 x1875872189592704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972198.182701:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791880 x1875872189592704/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972214 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972198.182764:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dc000. 02000000:00000010:2.0:1788972198.182771:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788972198.182778:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791880. 00000100:00000010:2.0:1788972199.186003:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0. 00000100:00000010:3.0:1788972199.187502:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e3c0. 00000100:00000010:3.0:1788972203.282516:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8792a00. 02000000:00000010:3.0:1788972203.282544:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972203.282573:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791500. 02000000:00000010:3.0:1788972203.282580:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67800. 00000100:00000010:3.0:1788972203.282594:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791880. 02000000:00000010:3.0:1788972203.282599:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 02000000:00000010:2.0:1788972203.282765:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dc000. 00000400:00000200:2.0:1788972203.282780:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972203.282786:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:2.0:1788972203.282795:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972203.282804:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189592960, portal 25 00000400:00000010:2.0:1788972203.282812:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b4c8. 00000100:00000200:2.0:1788972203.282851:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189592960, offset 0 00000400:00000200:2.0:1788972203.282865:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972203.282876:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972203.282885:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972203.282892:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972203.282898:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972203.282902:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972203.282905:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972203.282913:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [624:624:256:4294967295] 192.168.203.150@tcp LPNI seq info [624:624:8:4294967295] 00000400:00000200:2.0:1788972203.282919:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972203.282930:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972203.282937:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972203.282943:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972203.282964:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972203.282973:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972203.282979:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972203.282995:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972203.283001:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972203.283015:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ea80. 02000000:00000010:2.0:1788972203.283044:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 00000400:00000200:2.0:1788972203.283049:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972203.283052:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c08. 00000400:00000010:2.0:1788972203.283056:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972203.283059:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593216, portal 10 00000400:00000010:2.0:1788972203.283064:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b7f8. 00000100:00000200:2.0:1788972203.283066:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189593216, offset 0 00000400:00000200:2.0:1788972203.283070:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972203.283074:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972203.283077:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972203.283080:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972203.283082:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972203.283085:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972203.283087:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972203.283089:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [625:625:255:4294967295] 192.168.203.150@tcp LPNI seq info [625:625:7:4294967295] 00000400:00000200:2.0:1788972203.283093:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972203.283098:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972203.283103:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972203.283106:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63700. 00000800:00000200:2.0:1788972203.283111:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972203.283115:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972203.283117:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972203.283124:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972203.283146:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd000. 00000400:00000200:2.0:1788972203.283150:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972203.283153:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972203.283156:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972203.283159:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593088, portal 10 00000400:00000010:2.0:1788972203.283165:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b088. 00000100:00000200:2.0:1788972203.283167:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189593088, offset 0 00000400:00000200:2.0:1788972203.283172:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972203.283182:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972203.283186:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972203.283188:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972203.283190:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972203.283192:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972203.283195:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972203.283197:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [626:626:254:4294967295] 192.168.203.150@tcp LPNI seq info [626:626:6:4294967295] 00000400:00000200:2.0:1788972203.283201:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972203.283207:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972203.283212:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972203.283226:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63f00. 00000800:00000200:2.0:1788972203.283231:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972203.283236:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972203.283239:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63f00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972203.283244:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972203.283577:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972203.283585:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:3.0:1788972203.283593:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972203.283603:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972203.283611:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b4c8 00000400:00000010:3.0:1788972203.283614:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b4c8. 00000100:00000200:3.0:1788972203.283629:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8792a00 x1875872189592960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972203.283659:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972203.283662:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63700. 00000400:00000200:3.0:1788972203.283679:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972203.283683:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972203.283686:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b7f8 00000400:00000010:3.0:1788972203.283688:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b7f8. 00000100:00000200:3.0:1788972203.283695:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791880 x1875872189593216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972203.283747:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972203.283750:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63f00. 00000400:00000200:3.0:1788972203.283755:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972203.283759:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972203.283765:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b088 00000400:00000010:3.0:1788972203.283767:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b088. 00000100:00000200:3.0:1788972203.283775:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791500 x1875872189593088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972203.285964:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.285976:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972203.285986:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.285992:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.285999:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286002:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813680 00000400:00000200:3.0:1788972203.286009:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286016:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ec5 [1] + 224 00000400:00000200:3.0:1788972203.286032:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972203.286036:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972203.286054:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791880 x1875872189593216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972203.286097:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286101:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972203.286105:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286108:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286134:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286141:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813600 00000400:00000200:3.0:1788972203.286144:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.286148:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ed5 [1] + 224 00000400:00000200:3.0:1788972203.286158:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972203.286161:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972203.286168:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791500 x1875872189593088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972203.288067:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972203.288079:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:2.0:1788972203.288081:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972203.288094:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791500 x1875872189593088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972203.288105:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791500 x1875872189593088/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972203.288144:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c08. 00000400:00000200:2.0:1788972203.288147:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972203.288149:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972203.288154:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791880 x1875872189593216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972203.288163:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791880 x1875872189593216/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972203.288183:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd000. 02000000:00000010:2.0:1788972203.288208:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67800. 00000100:00000010:2.0:1788972203.288213:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791500. 02000000:00000010:2.0:1788972203.288228:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:2.0:1788972203.288231:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788972203.288235:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791880. 00000400:00000200:3.0:1788972203.288350:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.288359:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972203.288369:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.288375:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.288381:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.288386:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813580 00000400:00000200:3.0:1788972203.288392:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972203.288399:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1eb5 [1] + 224 00000400:00000200:3.0:1788972203.288415:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972203.288421:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972203.288434:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8792a00 x1875872189592960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972203.288645:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:2.0:1788972203.288650:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972203.288652:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972203.288660:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8792a00 x1875872189592960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972203.288669:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8792a00 x1875872189592960/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972219 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972203.288683:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dc000. 02000000:00000010:2.0:1788972203.288686:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:2.0:1788972203.288689:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8792a00. 00000100:00000010:3.0:1788972204.305971:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ea80. 00000100:00000010:2.0:1788972204.305971:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972208.402274:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972208.402298:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972208.402325:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972208.402331:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972208.402337:0:8:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972208.402342:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00. 02000000:00000010:2.0:1788972208.403400:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dc000. 00000400:00000200:2.0:1788972208.403414:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972208.403418:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:2.0:1788972208.403426:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972208.403433:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593344, portal 25 00000400:00000010:2.0:1788972208.403439:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972208.403441:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189593344, offset 0 00000400:00000200:2.0:1788972208.403451:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972208.403459:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972208.403477:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972208.403482:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972208.403487:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972208.403490:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972208.403493:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972208.403497:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [627:627:256:4294967295] 192.168.203.150@tcp LPNI seq info [627:627:8:4294967295] 00000400:00000200:2.0:1788972208.403502:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972208.403511:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972208.403517:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972208.403522:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63700. 00000800:00000200:2.0:1788972208.403529:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972208.403537:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972208.403541:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972208.403560:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972208.403564:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972208.403575:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed20. 02000000:00000010:2.0:1788972208.403586:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 00000400:00000200:2.0:1788972208.403590:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972208.403592:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c08. 00000400:00000010:2.0:1788972208.403596:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972208.403599:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593600, portal 10 00000400:00000010:2.0:1788972208.403603:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972208.403605:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189593600, offset 0 00000400:00000200:2.0:1788972208.403610:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972208.403613:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972208.403616:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972208.403619:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972208.403621:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972208.403623:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972208.403626:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972208.403628:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [628:628:255:4294967295] 192.168.203.150@tcp LPNI seq info [628:628:7:4294967295] 00000400:00000200:2.0:1788972208.403638:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972208.403644:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972208.403648:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972208.403651:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972208.403655:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972208.403659:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972208.403662:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972208.403667:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972208.403684:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd000. 00000400:00000200:2.0:1788972208.403687:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972208.403689:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972208.403693:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972208.403695:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593472, portal 10 00000400:00000010:2.0:1788972208.403699:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972208.403701:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189593472, offset 0 00000400:00000200:2.0:1788972208.403705:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972208.403709:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972208.403712:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972208.403715:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972208.403717:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972208.403719:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972208.403721:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972208.403724:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [629:629:254:4294967295] 192.168.203.150@tcp LPNI seq info [629:629:6:4294967295] 00000400:00000200:2.0:1788972208.403728:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972208.403743:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972208.403748:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972208.403751:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000800:00000200:2.0:1788972208.403755:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972208.403759:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972208.403761:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972208.403914:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972208.404082:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972208.404087:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63700. 00000400:00000200:2.0:1788972208.404092:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972208.404097:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972208.404102:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972208.404105:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972208.404113:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189593344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972224 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972208.404148:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972208.404150:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:2.0:1788972208.404153:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972208.404156:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972208.404159:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972208.404160:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972208.404165:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189593600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972208.404514:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972208.404517:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000200:2.0:1788972208.404521:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972208.404525:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972208.404529:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:2.0:1788972208.404531:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972208.404536:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189593472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972208.407415:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407425:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972208.407432:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407437:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407442:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407446:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813800 00000400:00000200:3.0:1788972208.407452:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407458:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ef5 [1] + 224 00000400:00000200:3.0:1788972208.407483:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972208.407488:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972208.407500:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189593600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972208.407537:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407540:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972208.407544:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407547:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407549:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407552:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813780 00000400:00000200:3.0:1788972208.407556:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407559:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f05 [1] + 224 00000400:00000200:3.0:1788972208.407565:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972208.407569:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972208.407574:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189593472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972208.407587:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407590:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972208.407593:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407596:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407599:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407601:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813700 00000400:00000200:3.0:1788972208.407604:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972208.407607:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ee5 [1] + 224 00000400:00000200:3.0:1788972208.407614:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972208.407617:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972208.407622:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189593344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972208.408085:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972208.408091:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972208.408094:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972208.408100:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189593472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972208.408109:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189593472/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972208.408144:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:2.0:1788972208.408147:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972208.408149:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972208.408153:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189593344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972208.408161:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189593344/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972208.408171:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c08. 00000400:00000200:2.0:1788972208.408173:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:2.0:1788972208.408175:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972208.408188:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189593600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972208.408196:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189593600/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972224 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972208.408213:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd000. 02000000:00000010:2.0:1788972208.408395:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972208.408402:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972208.408406:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dc000. 02000000:00000010:2.0:1788972208.408408:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972208.408410:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972208.408413:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:2.0:1788972208.408415:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00. 00000100:00000010:2.0:1788972208.408417:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 00000100:00000010:2.0:1788972209.426003:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0. 00000100:00000010:3.0:1788972209.426149:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed20. 00000100:00000010:0.0:1788972213.526089:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800. 02000000:00000010:0.0:1788972213.526125:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffb00. 00000100:00000010:0.0:1788972213.526173:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788972213.526180:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff400. 00000100:00000010:0.0:1788972213.526186:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972213.526189:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff000. 02000000:00000010:1.0:1788972213.526341:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d40d8e00. 00000400:00000200:1.0:1788972213.526352:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972213.526356:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972213.526437:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788972213.526446:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593728, portal 25 00000400:00000010:1.0:1788972213.526452:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525dd0. 00000100:00000200:1.0:1788972213.526455:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189593728, offset 0 00000400:00000200:1.0:1788972213.526464:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972213.526472:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972213.526480:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972213.526485:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972213.526489:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972213.526492:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972213.526494:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972213.526498:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [630:630:256:4294967295] 192.168.203.150@tcp LPNI seq info [630:630:8:4294967295] 00000400:00000200:1.0:1788972213.526501:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972213.526512:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972213.526526:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972213.526531:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b600. 00000800:00000200:1.0:1788972213.526537:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972213.526544:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972213.526549:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972213.526574:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972213.526580:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467240. 00000100:00000010:1.0:1788972213.526590:0:102141:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467d80. 02000000:00000010:1.0:1788972213.526602:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1302200. 00000400:00000200:1.0:1788972213.526606:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972213.526608:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972213.526612:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972213.526614:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593984, portal 10 00000400:00000010:1.0:1788972213.526620:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c75256e8. 00000100:00000200:1.0:1788972213.526622:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189593984, offset 0 00000400:00000200:1.0:1788972213.526625:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972213.526629:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972213.526632:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972213.526634:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972213.526637:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972213.526639:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972213.526640:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972213.526643:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [631:631:255:4294967295] 192.168.203.150@tcp LPNI seq info [631:631:7:4294967295] 00000400:00000200:1.0:1788972213.526646:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972213.526651:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972213.526655:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972213.526659:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b500. 00000800:00000200:1.0:1788972213.526662:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972213.526666:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972213.526668:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b500 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972213.526680:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972213.526697:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600. 00000400:00000200:1.0:1788972213.526701:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972213.526703:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972213.526707:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788972213.526710:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189593856, portal 10 00000400:00000010:1.0:1788972213.526714:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525cc0. 00000100:00000200:1.0:1788972213.526716:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189593856, offset 0 00000400:00000200:1.0:1788972213.526720:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972213.526724:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972213.526727:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972213.526729:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972213.526732:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972213.526734:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972213.526736:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972213.526739:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [632:632:254:4294967295] 192.168.203.150@tcp LPNI seq info [632:632:6:4294967295] 00000400:00000200:1.0:1788972213.526742:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972213.526747:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972213.526751:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972213.526754:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b900. 00000800:00000200:1.0:1788972213.526758:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972213.526762:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972213.526764:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972213.526769:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972213.526906:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972213.526910:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b600. 00000400:00000200:2.0:1788972213.526916:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972213.526921:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972213.526926:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525dd0 00000400:00000010:2.0:1788972213.526934:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525dd0. 00000100:00000200:2.0:1788972213.526943:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b800 x1875872189593728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972229 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972213.526965:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972213.526968:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b500. 00000400:00000200:2.0:1788972213.526971:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972213.526974:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972213.526976:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c75256e8 00000400:00000010:2.0:1788972213.526978:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c75256e8. 00000100:00000200:2.0:1788972213.526982:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189593984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972213.527049:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972213.527052:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b900. 00000400:00000200:2.0:1788972213.527055:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972213.527059:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972213.527061:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525cc0 00000400:00000010:2.0:1788972213.527063:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525cc0. 00000100:00000200:2.0:1788972213.527068:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88000 x1875872189593856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972213.531383:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531394:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972213.531403:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531407:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531412:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531416:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813980 00000400:00000200:3.0:1788972213.531425:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531432:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f1d [1] + 224 00000400:00000200:3.0:1788972213.531443:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972213.531448:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972213.531461:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189593984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972213.531499:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531503:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972213.531508:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531510:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531513:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531515:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813900 00000400:00000200:3.0:1788972213.531518:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531521:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f25 [1] + 224 00000400:00000200:3.0:1788972213.531527:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972213.531530:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972213.531536:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88000 x1875872189593856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972213.531550:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531554:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972213.531558:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531560:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531562:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531565:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813880 00000400:00000200:3.0:1788972213.531582:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972213.531586:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f15 [1] + 224 00000400:00000200:3.0:1788972213.531596:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972213.531599:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972213.531605:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b800 x1875872189593728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972213.531751:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972213.531756:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:1.0:1788972213.531759:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:1.0:1788972213.531765:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88000 x1875872189593856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972213.531774:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88000 x1875872189593856/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972213.531845:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972213.531849:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:1.0:1788972213.531851:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000100:00000200:1.0:1788972213.531855:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b800 x1875872189593728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972213.531864:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b800 x1875872189593728/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972213.531875:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972213.531878:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972213.531879:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972213.531883:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189593984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972213.531891:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189593984/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972229 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972213.531905:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600. 02000000:00000010:1.0:1788972213.531914:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff400. 00000100:00000010:1.0:1788972213.531918:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88000. 02000000:00000010:1.0:1788972213.531922:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d40d8e00. 02000000:00000010:1.0:1788972213.531926:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffb00. 00000100:00000010:1.0:1788972213.531928:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800. 02000000:00000010:1.0:1788972213.531931:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1302200. 02000000:00000010:1.0:1788972213.531933:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff000. 00000100:00000010:1.0:1788972213.531936:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000100:00000010:3.0:1788972214.545964:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467d80. 00000100:00000010:2.0:1788972214.546091:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467240. 00000100:00000010:2.0:1788972218.642613:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972218.642645:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00. 00000100:00000010:2.0:1788972218.642694:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d500. 02000000:00000010:2.0:1788972218.642701:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63d00. 00000100:00000010:2.0:1788972218.642708:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972218.642712:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 02000000:00000010:2.0:1788972218.642778:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dce00. 00000400:00000200:2.0:1788972218.642790:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972218.642796:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126c08. 00000400:00000010:2.0:1788972218.642805:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bbb0. 00000100:00000200:2.0:1788972218.642843:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594240, portal 10 00000400:00000010:2.0:1788972218.642873:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972218.642877:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189594240, offset 0 00000400:00000200:2.0:1788972218.642889:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972218.643005:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.643016:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972218.643023:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972218.643027:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972218.643032:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972218.643035:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972218.643039:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [633:633:256:4294967295] 192.168.203.150@tcp LPNI seq info [633:633:8:4294967295] 00000400:00000200:2.0:1788972218.643043:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.643053:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972218.643059:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972218.643065:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000800:00000200:2.0:1788972218.643072:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972218.643081:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972218.643096:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972218.643122:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:2.0:1788972218.643129:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651e0c0. 00000100:00000010:2.0:1788972218.643141:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c651ed80. 02000000:00000010:2.0:1788972218.643170:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dc000. 00000400:00000200:2.0:1788972218.643174:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972218.643177:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:2.0:1788972218.643180:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972218.643183:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594112, portal 25 00000400:00000010:2.0:1788972218.643187:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972218.643189:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189594112, offset 0 00000400:00000200:2.0:1788972218.643194:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972218.643198:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.643201:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972218.643204:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972218.643212:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972218.643215:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972218.643217:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972218.643220:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [634:634:255:4294967295] 192.168.203.150@tcp LPNI seq info [634:634:7:4294967295] 00000400:00000200:2.0:1788972218.643225:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.643231:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972218.643236:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972218.643240:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63400. 00000800:00000200:2.0:1788972218.643244:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972218.643248:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972218.643251:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972218.643255:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972218.643268:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd000. 00000400:00000200:2.0:1788972218.643272:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972218.643274:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000400:00000010:2.0:1788972218.643277:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972218.643280:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594368, portal 10 00000400:00000010:2.0:1788972218.643284:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972218.643287:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189594368, offset 0 00000400:00000200:2.0:1788972218.643291:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:2.0:1788972218.643294:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.643297:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972218.643300:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972218.643302:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972218.643304:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972218.643306:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972218.643309:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [635:635:254:4294967295] 192.168.203.150@tcp LPNI seq info [635:635:6:4294967295] 00000400:00000200:2.0:1788972218.643312:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.643318:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972218.643322:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972218.643330:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63700. 00000800:00000200:2.0:1788972218.643334:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972218.643338:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972218.643341:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972218.643347:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972218.643407:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972218.643413:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000200:2.0:1788972218.643420:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972218.643426:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972218.643433:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b660 00000400:00000010:2.0:1788972218.643436:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b660. 00000100:00000200:2.0:1788972218.643447:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d500 x1875872189594240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972218.643473:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972218.643475:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63400. 00000400:00000200:2.0:1788972218.643478:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972218.643481:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972218.643483:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:2.0:1788972218.643485:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972218.643490:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189594112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972234 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972218.643571:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972218.643574:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63700. 00000400:00000200:2.0:1788972218.643576:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972218.643580:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972218.643582:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972218.643584:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972218.643589:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189594368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972218.647190:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647203:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972218.647213:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647219:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647225:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647230:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813a00 00000400:00000200:3.0:1788972218.647236:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647243:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f3d [1] + 224 00000400:00000200:3.0:1788972218.647258:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972218.647263:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972218.647280:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189594112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972218.647368:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647372:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972218.647377:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647379:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647382:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647385:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813b00 00000400:00000200:3.0:1788972218.647389:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972218.647394:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f4d [1] + 224 00000400:00000200:3.0:1788972218.647402:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972218.647405:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972218.647435:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189594368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972218.647665:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:2.0:1788972218.647673:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:2.0:1788972218.647684:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972218.647691:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189594112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972218.647699:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189594112/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972218.647729:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972218.647732:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972218.647734:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972218.647737:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189594368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972218.647744:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189594368/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972218.647778:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dc000. 02000000:00000010:2.0:1788972218.647787:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00. 00000100:00000010:2.0:1788972218.647792:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972218.647796:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd000. 02000000:00000010:2.0:1788972218.647798:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972218.647800:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 00000400:00000200:2.0:1788972218.648860:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.648868:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972218.648875:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.648878:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.648882:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.648886:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813a80 00000400:00000200:2.0:1788972218.648892:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972218.648906:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f2d [1] + 224 00000400:00000200:2.0:1788972218.648921:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972218.648926:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972218.648939:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d500 x1875872189594240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972218.649358:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126c08. 00000400:00000200:3.0:1788972218.649365:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bbb0 00000400:00000010:3.0:1788972218.649367:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bbb0. 00000100:00000200:3.0:1788972218.649374:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d500 x1875872189594240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972218.649383:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d500 x1875872189594240/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972234 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972218.649416:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dce00. 02000000:00000010:3.0:1788972218.649421:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63d00. 00000100:00000010:3.0:1788972218.649426:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d500. 00000100:00000010:2.0:1788972219.666342:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651e0c0. 00000100:00000010:3.0:1788972219.666525:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c651ed80. 00000100:00000010:0.0:1788972223.762324:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972223.762356:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff500. 00000100:00000010:0.0:1788972223.762392:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88e00. 02000000:00000010:0.0:1788972223.762397:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffe00. 00000100:00000010:0.0:1788972223.762407:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972223.762410:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff200. 02000000:00000010:0.0:1788972223.762481:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:0.0:1788972223.762491:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972223.762496:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972223.762504:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972223.762513:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594496, portal 25 00000400:00000010:0.0:1788972223.762520:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8dd0. 00000100:00000200:0.0:1788972223.762524:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189594496, offset 0 00000400:00000200:0.0:1788972223.762535:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972223.762545:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972223.762554:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972223.762561:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972223.762566:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972223.762572:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972223.762575:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972223.762580:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [636:636:256:4294967295] 192.168.203.150@tcp LPNI seq info [636:636:8:4294967295] 00000400:00000200:0.0:1788972223.762585:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972223.762592:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d8835600. 00000400:00000200:1.0:1788972223.762602:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972223.762602:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000400:00000010:1.0:1788972223.762608:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972223.762614:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000800:00000200:0.0:1788972223.762614:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000100:00000200:1.0:1788972223.762620:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594624, portal 10 00000800:00000010:0.0:1788972223.762623:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cff800. 00000400:00000010:1.0:1788972223.762627:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525198. 00000800:00000200:0.0:1788972223.762634:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000100:00000200:1.0:1788972223.762648:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189594624, offset 0 00000800:00000200:0.0:1788972223.762656:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000400:00000200:1.0:1788972223.762657:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972223.762663:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:0.0:1788972223.762665:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cff800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972223.762669:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972223.762673:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972223.762677:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972223.762680:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972223.762684:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972223.762685:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972223.762691:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [637:637:255:4294967295] 192.168.203.150@tcp LPNI seq info [637:637:7:4294967295] 00000100:00000010:0.0:1788972223.762694:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5720. 00000400:00000200:1.0:1788972223.762697:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972223.762705:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972223.762710:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000100:00000010:0.0:1788972223.762712:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c52a0. 00000800:00000010:1.0:1788972223.762715:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b400. 00000800:00000200:1.0:1788972223.762721:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972223.762725:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972223.762728:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b400 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972223.762735:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972223.763513:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972223.763521:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cff800. 00000400:00000200:3.0:1788972223.763527:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972223.763535:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972223.763542:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8dd0 00000400:00000010:3.0:1788972223.763545:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8dd0. 00000100:00000200:3.0:1788972223.763556:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189594496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972239 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972223.763629:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972223.763632:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b400. 00000400:00000200:3.0:1788972223.763636:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972223.763640:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972223.763643:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525198 00000400:00000010:3.0:1788972223.763645:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525198. 00000100:00000200:3.0:1788972223.763651:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88e00 x1875872189594624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972223.764530:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a800. 00000400:00000200:0.0:1788972223.764543:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972223.764547:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4e70. 00000400:00000010:0.0:1788972223.764554:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58. 00000100:00000200:0.0:1788972223.764561:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594752, portal 10 00000400:00000010:0.0:1788972223.764569:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a84c8. 00000100:00000200:0.0:1788972223.764573:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189594752, offset 0 00000400:00000200:0.0:1788972223.764581:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972223.764588:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972223.764712:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972223.764717:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972223.764721:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:0.0:1788972223.764725:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972223.764727:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972223.764731:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [638:638:256:4294967295] 192.168.203.150@tcp LPNI seq info [638:638:8:4294967295] 00000400:00000200:0.0:1788972223.764736:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972223.764746:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972223.764752:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972223.764758:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cffc00. 00000800:00000200:0.0:1788972223.764765:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972223.764772:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972223.764777:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cffc00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972223.764872:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972223.766392:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972223.766401:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cffc00. 00000400:00000200:2.0:1788972223.766409:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972223.766418:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972223.766442:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a84c8 00000400:00000010:2.0:1788972223.766446:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a84c8. 00000100:00000200:2.0:1788972223.766458:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189594752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972223.768847:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.768888:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972223.768899:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.768905:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.768911:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.768915:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813c00 00000400:00000200:3.0:1788972223.768922:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.768929:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f65 [1] + 224 00000400:00000200:3.0:1788972223.768946:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972223.768951:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972223.768966:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88e00 x1875872189594624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972223.769137:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972223.769144:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972223.769147:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972223.769156:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88e00 x1875872189594624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972223.769168:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88e00 x1875872189594624/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972223.769207:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d8835600. 02000000:00000010:1.0:1788972223.769217:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffe00. 00000100:00000010:1.0:1788972223.769222:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88e00. 00000400:00000200:2.0:1788972223.771909:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972223.771922:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972223.771932:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972223.771939:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972223.771944:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972223.771948:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813c80 00000400:00000200:2.0:1788972223.771955:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972223.771962:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f6d [1] + 224 00000400:00000200:2.0:1788972223.771977:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972223.771982:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972223.771998:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189594752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972223.776185:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.776199:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972223.776211:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.776217:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.776225:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.776229:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813b80 00000400:00000200:3.0:1788972223.776234:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972223.776241:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f5d [1] + 224 00000400:00000200:3.0:1788972223.776258:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972223.776514:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972223.776933:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189594496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972223.777441:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4e70. 00000400:00000200:0.0:1788972223.777448:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58 00000400:00000010:0.0:1788972223.777452:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58. 00000100:00000200:0.0:1788972223.777483:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189594752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972223.777494:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189594752/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972223.777534:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a800. 02000000:00000010:0.0:1788972223.777593:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff200. 00000100:00000010:0.0:1788972223.777600:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000400:00000010:0.0:1788972223.777610:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972223.777612:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788972223.777614:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972223.777619:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189594496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972223.777627:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189594496/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972239 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972223.777639:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:0.0:1788972223.777642:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff500. 00000100:00000010:0.0:1788972223.777644:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 00000100:00000010:2.0:1788972224.786491:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5720. 00000100:00000010:3.0:1788972224.789889:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c52a0. 00000100:00000010:2.0:1788972228.882854:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972228.882887:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972228.882919:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840dc00. 02000000:00000010:2.0:1788972228.882925:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63f00. 00000100:00000010:2.0:1788972228.882932:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972228.882936:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63700. 02000000:00000010:3.0:1788972228.883045:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ec00. 00000400:00000200:3.0:1788972228.883059:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972228.883065:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436d10. 00000400:00000010:3.0:1788972228.883074:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873f68. 00000100:00000200:3.0:1788972228.883084:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189594880, portal 25 00000400:00000010:3.0:1788972228.883091:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873e58. 00000100:00000200:3.0:1788972228.883111:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189594880, offset 0 00000400:00000200:3.0:1788972228.883124:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972228.883135:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.883145:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972228.883151:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972228.883155:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972228.883160:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972228.883163:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972228.883167:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [639:639:256:4294967295] 192.168.203.150@tcp LPNI seq info [639:639:8:4294967295] 00000400:00000200:3.0:1788972228.883171:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.883183:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972228.883189:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972228.883195:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67e00. 00000800:00000200:3.0:1788972228.883202:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972228.883211:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972228.883217:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67e00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972228.883239:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972228.883244:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09d20. 00000100:00000010:3.0:1788972228.883261:0:102142:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09060. 02000000:00000010:3.0:1788972228.883275:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e600. 00000400:00000200:3.0:1788972228.883278:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972228.883281:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972228.883285:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873088. 00000100:00000200:3.0:1788972228.883287:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595136, portal 10 00000400:00000010:3.0:1788972228.883293:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873cc0. 00000100:00000200:3.0:1788972228.883295:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189595136, offset 0 00000400:00000200:3.0:1788972228.883299:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972228.883302:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.883305:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972228.883308:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972228.883311:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972228.883328:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972228.883331:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972228.883335:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [640:640:255:4294967295] 192.168.203.150@tcp LPNI seq info [640:640:7:4294967295] 00000400:00000200:3.0:1788972228.883340:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.883347:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972228.883352:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972228.883356:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67c00. 00000800:00000200:3.0:1788972228.883360:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972228.883364:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972228.883367:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972228.883373:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972228.883394:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107f400. 00000400:00000200:3.0:1788972228.883398:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972228.883401:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436840. 00000400:00000010:3.0:1788972228.883405:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972228.883409:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595008, portal 10 00000800:00000200:2.0:1788972228.883409:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972228.883421:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67e00. 00000400:00000010:3.0:1788972228.883424:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48732a8. 00000100:00000200:3.0:1788972228.883428:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189595008, offset 0 00000400:00000200:2.0:1788972228.883430:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972228.883435:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972228.883441:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972228.883442:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972228.883447:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972228.883451:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972228.883453:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972228.883455:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972228.883457:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972228.883460:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [641:641:254:4294967295] 192.168.203.150@tcp LPNI seq info [641:641:6:4294967295] 00000400:00000200:3.0:1788972228.883464:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.883469:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972228.883473:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972228.883477:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972228.883478:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873e58 00000400:00000010:2.0:1788972228.883482:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873e58. 00000800:00000200:3.0:1788972228.883489:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972228.883494:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972228.883498:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:2.0:1788972228.883503:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189594880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972244 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972228.883506:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972228.883540:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972228.883543:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67c00. 00000400:00000200:2.0:1788972228.883546:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972228.883550:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972228.883554:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873cc0 00000400:00000010:2.0:1788972228.883556:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873cc0. 00000100:00000200:2.0:1788972228.883562:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189595136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972228.884090:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972228.884094:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972228.884099:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972228.884104:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972228.884107:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48732a8 00000400:00000010:2.0:1788972228.884109:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48732a8. 00000100:00000200:2.0:1788972228.884116:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840dc00 x1875872189595008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972228.889764:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889777:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972228.889787:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889793:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889799:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889804:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813d00 00000400:00000200:3.0:1788972228.889810:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889855:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f75 [1] + 224 00000400:00000200:3.0:1788972228.889869:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972228.889873:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972228.889891:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189594880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1788972228.889937:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889940:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972228.889945:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889947:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889949:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889953:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813e00 00000400:00000200:3.0:1788972228.889956:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972228.889960:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f85 [1] + 224 00000400:00000200:3.0:1788972228.889968:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972228.889971:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972228.889988:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189595136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972228.890167:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436d10. 00000400:00000200:2.0:1788972228.890174:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873f68 00000400:00000010:2.0:1788972228.890177:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873f68. 00000100:00000200:2.0:1788972228.890185:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189594880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972228.890193:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189594880/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972228.890223:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:2.0:1788972228.890257:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873088 00000400:00000010:2.0:1788972228.890258:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873088. 00000100:00000200:2.0:1788972228.890264:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189595136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972228.890272:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189595136/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972228.890298:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ec00. 02000000:00000010:2.0:1788972228.890305:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972228.890311:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972228.890315:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e600. 02000000:00000010:2.0:1788972228.890317:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63700. 00000100:00000010:2.0:1788972228.890324:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 00000400:00000200:2.0:1788972228.891543:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972228.891553:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972228.891572:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972228.891577:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972228.891581:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972228.891586:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813d80 00000400:00000200:2.0:1788972228.891591:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972228.891598:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1f95 [1] + 224 00000400:00000200:2.0:1788972228.891614:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972228.892452:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972228.892469:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840dc00 x1875872189595008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972228.893671:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436840. 00000400:00000200:3.0:1788972228.893704:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972228.893708:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972228.893720:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840dc00 x1875872189595008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972228.893733:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840dc00 x1875872189595008/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972244 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972228.893776:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107f400. 02000000:00000010:3.0:1788972228.893783:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63f00. 00000100:00000010:3.0:1788972228.893789:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840dc00. 00000100:00000010:2.0:1788972229.905997:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09d20. 00000100:00000010:3.0:1788972229.906138:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09060. 00000100:00000010:2.0:1788972234.002136:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d880. 02000000:00000010:2.0:1788972234.002159:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63700. 00000100:00000010:2.0:1788972234.002184:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840d180. 02000000:00000010:2.0:1788972234.002189:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63c00. 00000100:00000010:2.0:1788972234.002195:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c840e680. 02000000:00000010:2.0:1788972234.002215:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73d0f63400. 02000000:00000010:3.0:1788972234.002478:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107fa00. 00000400:00000200:3.0:1788972234.002489:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972234.002494:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436840. 00000400:00000010:3.0:1788972234.002500:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972234.002508:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595264, portal 25 00000400:00000010:3.0:1788972234.002513:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873198. 00000100:00000200:3.0:1788972234.002517:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189595264, offset 0 00000400:00000200:3.0:1788972234.002526:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972234.002547:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972234.002554:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972234.002559:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972234.002563:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972234.002567:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972234.002570:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972234.002573:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [642:642:256:4294967295] 192.168.203.150@tcp LPNI seq info [642:642:8:4294967295] 00000400:00000200:3.0:1788972234.002578:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972234.002588:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972234.002594:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972234.002599:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 00000800:00000200:3.0:1788972234.002605:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972234.002619:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972234.002624:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972234.002645:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972234.002651:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09060. 00000100:00000010:3.0:1788972234.002665:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09240. 02000000:00000010:3.0:1788972234.002676:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107ea00. 00000400:00000200:3.0:1788972234.002680:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972234.002683:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436cb8. 00000400:00000010:3.0:1788972234.002686:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788972234.002689:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595520, portal 10 00000400:00000010:3.0:1788972234.002694:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873ee0. 00000100:00000200:3.0:1788972234.002697:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189595520, offset 0 00000400:00000200:3.0:1788972234.002701:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972234.002705:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972234.002709:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972234.002711:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972234.002714:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972234.002716:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972234.002719:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972234.002721:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [643:643:255:4294967295] 192.168.203.150@tcp LPNI seq info [643:643:7:4294967295] 00000400:00000200:3.0:1788972234.002725:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972234.002731:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972234.002735:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972234.002739:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67100. 00000800:00000200:3.0:1788972234.002742:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972234.002747:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972234.002750:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972234.002755:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:3.0:1788972234.002773:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e400. 00000400:00000200:3.0:1788972234.002776:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972234.002784:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972234.002788:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972234.002792:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595392, portal 10 00000400:00000010:3.0:1788972234.002796:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873440. 00000100:00000200:3.0:1788972234.002798:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189595392, offset 0 00000400:00000200:3.0:1788972234.002803:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972234.002806:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972234.002809:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972234.002812:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972234.002851:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972234.002854:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972234.002856:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972234.002859:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [644:644:254:4294967295] 192.168.203.150@tcp LPNI seq info [644:644:6:4294967295] 00000400:00000200:3.0:1788972234.002863:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972234.002869:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972234.002874:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972234.002877:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67c00. 00000800:00000200:3.0:1788972234.002881:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:3.0:1788972234.002885:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972234.002887:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67c00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1788972234.002892:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972234.003092:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972234.003097:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972234.003103:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972234.003107:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972234.003114:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873198 00000400:00000010:2.0:1788972234.003116:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873198. 00000100:00000200:2.0:1788972234.003126:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d880 x1875872189595264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972250 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972234.003149:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972234.003151:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67100. 00000400:00000200:2.0:1788972234.003153:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972234.003156:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972234.003159:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873ee0 00000400:00000010:2.0:1788972234.003160:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873ee0. 00000100:00000200:2.0:1788972234.003164:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840e680 x1875872189595520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972234.003380:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972234.003383:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67c00. 00000400:00000200:2.0:1788972234.003386:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972234.003391:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972234.003394:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873440 00000400:00000010:2.0:1788972234.003396:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873440. 00000100:00000200:2.0:1788972234.003401:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c840d180 x1875872189595392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972234.009865:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.009875:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972234.009884:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.009888:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.009894:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.009898:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887813e80 00000400:00000200:2.0:1788972234.009903:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.009991:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fa5 [1] + 224 00000400:00000200:2.0:1788972234.010013:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972234.010018:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972234.010031:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d880 x1875872189595264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972234.010118:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010122:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972234.010126:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010128:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010131:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010134:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813f80 00000400:00000200:2.0:1788972234.010138:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010142:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fb5 [1] + 224 00000400:00000200:2.0:1788972234.010147:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972234.010151:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972234.010157:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840e680 x1875872189595520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972234.010171:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010174:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972234.010178:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010181:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010183:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010186:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887813f00 00000400:00000200:2.0:1788972234.010189:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972234.010193:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fc5 [1] + 224 00000400:00000200:2.0:1788972234.010202:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972234.010206:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972234.010212:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c840d180 x1875872189595392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972234.012614:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436840. 00000400:00000200:3.0:1788972234.012625:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873220 00000400:00000010:3.0:1788972234.012627:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873220. 00000100:00000200:3.0:1788972234.012640:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d880 x1875872189595264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972234.012651:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d880 x1875872189595264/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1788972234.012675:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436cb8. 00000400:00000200:3.0:1788972234.012678:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8 00000400:00000010:3.0:1788972234.012679:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788972234.012700:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840e680 x1875872189595520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972234.012708:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840e680 x1875872189595520/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972234.012729:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107fa00. 02000000:00000010:3.0:1788972234.012736:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63700. 00000100:00000010:3.0:1788972234.012741:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d880. 02000000:00000010:3.0:1788972234.012745:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107ea00. 02000000:00000010:3.0:1788972234.012748:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63400. 00000100:00000010:3.0:1788972234.012750:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840e680. 00000400:00000010:3.0:1788972234.012758:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972234.012760:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:3.0:1788972234.012761:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972234.012766:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c840d180 x1875872189595392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972234.012774:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c840d180 x1875872189595392/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972250 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972234.012786:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e400. 02000000:00000010:3.0:1788972234.012789:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73d0f63c00. 00000100:00000010:3.0:1788972234.012791:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c840d180. 00000100:00000010:2.0:1788972235.026127:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09060. 00000100:00000010:3.0:1788972235.026669:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09240. 00000100:00000010:0.0:1788972239.121981:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a89500. 02000000:00000010:0.0:1788972239.122067:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff500. 00000100:00000010:0.0:1788972239.122482:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972239.122489:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff200. 00000100:00000010:0.0:1788972239.122498:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b800. 02000000:00000010:0.0:1788972239.122503:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff300. 02000000:00000010:0.0:1788972239.122533:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d094a200. 00000400:00000200:0.0:1788972239.122542:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:0.0:1788972239.122546:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000010:0.0:1788972239.122553:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972239.122559:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595776, portal 10 02000000:00000010:1.0:1788972239.122560:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d1302200. 00000400:00000010:0.0:1788972239.122568:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73d13a8e58. 00000400:00000200:1.0:1788972239.122569:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:0.0:1788972239.122574:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189595776, offset 0 00000400:00000010:1.0:1788972239.122576:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972239.122590:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000400:00000200:0.0:1788972239.122592:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000100:00000200:1.0:1788972239.122596:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595648, portal 25 00000400:00000010:1.0:1788972239.122602:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525550. 00000400:00000200:0.0:1788972239.122603:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000200:1.0:1788972239.122609:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189595648, offset 0 00000400:00000200:0.0:1788972239.122613:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:0.0:1788972239.122621:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1788972239.122626:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972239.122629:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:0.0:1788972239.122630:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1788972239.122635:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:0.0:1788972239.122639:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [645:645:256:4294967295] 192.168.203.150@tcp LPNI seq info [645:645:8:4294967295] 00000400:00000200:0.0:1788972239.122643:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:0.0:1788972239.122652:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:0.0:1788972239.122657:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:0.0:1788972239.122662:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c7cfff00. 00000800:00000200:0.0:1788972239.122668:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:0.0:1788972239.122675:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:0.0:1788972239.122679:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c7cfff00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1788972239.122764:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:0.0:1788972239.122770:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5960. 00000100:00000010:0.0:1788972239.122784:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d40c5ae0. 00000800:00000200:2.0:1788972239.122872:0:102134:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972239.122878:0:102134:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c7cfff00. 00000400:00000200:2.0:1788972239.122883:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972239.122889:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972239.122895:0:102134:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a8e58 00000400:00000010:2.0:1788972239.122898:0:102134:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a8e58. 00000100:00000200:2.0:1788972239.122907:0:102134:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189595776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1788972239.123241:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972239.123246:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972239.123250:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972239.123253:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972239.123255:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972239.123257:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972239.123316:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [646:646:256:4294967295] 192.168.203.150@tcp LPNI seq info [646:646:8:4294967295] 00000400:00000200:1.0:1788972239.123320:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972239.123328:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972239.123332:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972239.123337:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1bb00. 00000800:00000200:1.0:1788972239.123341:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972239.123346:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972239.123349:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1bb00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972239.123366:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972239.123380:0:102141:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d40d8e00. 00000400:00000200:1.0:1788972239.123384:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972239.123387:0:102141:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972239.123391:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525330. 00000100:00000200:1.0:1788972239.123395:0:102141:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189595904, portal 10 00000400:00000010:1.0:1788972239.123400:0:102141:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788972239.123402:0:102141:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189595904, offset 0 00000400:00000200:1.0:1788972239.123407:0:102141:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972239.123410:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972239.123414:0:102141:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972239.123417:0:102141:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972239.123425:0:102141:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972239.123428:0:102141:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972239.123430:0:102141:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972239.123433:0:102141:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [647:647:255:4294967295] 192.168.203.150@tcp LPNI seq info [647:647:7:4294967295] 00000400:00000200:1.0:1788972239.123437:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972239.123443:0:102141:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972239.123446:0:102141:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972239.123448:0:102141:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b000. 00000800:00000200:1.0:1788972239.123451:0:102141:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972239.123454:0:102141:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972239.123456:0:102141:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972239.123459:0:102141:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:2.0:1788972239.123508:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972239.123553:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1bb00. 00000400:00000200:2.0:1788972239.123557:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972239.123561:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972239.123565:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525550 00000400:00000010:2.0:1788972239.123566:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525550. 00000100:00000200:2.0:1788972239.123573:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a89500 x1875872189595648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972255 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.0:1788972239.125994:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1788972239.126003:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b000. 00000400:00000200:2.0:1788972239.126009:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972239.126018:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972239.126025:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220 00000400:00000010:2.0:1788972239.126028:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220. 00000100:00000200:2.0:1788972239.126042:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b800 x1875872189595904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972239.126645:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126651:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972239.126668:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126672:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126677:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126693:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887814080 00000400:00000200:2.0:1788972239.126698:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126703:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fd5 [1] + 224 00000400:00000200:2.0:1788972239.126713:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972239.126717:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972239.126725:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189595776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972239.126757:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126760:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972239.126764:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126766:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126768:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126771:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887814100 00000400:00000200:2.0:1788972239.126774:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126777:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fe5 [1] + 224 00000400:00000200:2.0:1788972239.126783:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972239.126785:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972239.126790:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b800 x1875872189595904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972239.126892:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126896:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972239.126900:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126902:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126905:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126908:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887814000 00000400:00000200:2.0:1788972239.126911:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972239.126915:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fdd [1] + 224 00000400:00000200:2.0:1788972239.126923:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972239.126931:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972239.126943:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a89500 x1875872189595648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1788972239.127084:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73d4ff4bb0. 00000400:00000200:0.0:1788972239.127090:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73d13a83b8 00000400:00000010:0.0:1788972239.127092:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73d13a83b8. 00000100:00000200:0.0:1788972239.127099:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189595776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1788972239.127106:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189595776/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1788972239.127135:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d094a200. 02000000:00000010:0.0:1788972239.127141:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff200. 00000100:00000010:0.0:1788972239.127145:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000400:00000010:1.0:1788972239.128836:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972239.128847:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525330 00000400:00000010:1.0:1788972239.128850:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525330. 00000100:00000200:1.0:1788972239.128860:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b800 x1875872189595904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972239.128869:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b800 x1875872189595904/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972239.128894:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d40d8e00. 02000000:00000010:1.0:1788972239.128900:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff300. 00000100:00000010:1.0:1788972239.128904:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b800. 00000400:00000010:1.0:1788972239.128911:0:102141:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972239.128914:0:102141:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972239.128915:0:102141:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972239.128920:0:102141:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a89500 x1875872189595648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972239.128928:0:102141:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a89500 x1875872189595648/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972255 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972239.128938:0:102141:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d1302200. 02000000:00000010:1.0:1788972239.128957:0:102141:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff500. 00000100:00000010:1.0:1788972239.128969:0:102141:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a89500. 00000100:00000010:2.0:1788972240.145909:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5960. 00000100:00000010:3.0:1788972240.146253:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d40c5ae0. 00000100:00000010:3.0:1788972244.242127:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790a80. 02000000:00000010:3.0:1788972244.242150:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67e00. 00000100:00000010:3.0:1788972244.242193:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8790e00. 02000000:00000010:3.0:1788972244.242199:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972244.242206:0:114281:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73c8791c00. 02000000:00000010:3.0:1788972244.242219:0:114281:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c4c67100. 02000000:00000010:3.0:1788972244.242249:0:102143:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d107e000. 00000400:00000200:3.0:1788972244.242259:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:3.0:1788972244.242279:0:102143:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c4436e18. 00000400:00000010:3.0:1788972244.242286:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972244.242294:0:102143:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189596160, portal 10 00000400:00000010:3.0:1788972244.242301:0:102143:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788972244.242304:0:102143:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189596160, offset 0 00000400:00000200:3.0:1788972244.242312:0:102143:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972244.242321:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972244.242328:0:102143:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:3.0:1788972244.242333:0:102143:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1788972244.242337:0:102143:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972244.242341:0:102143:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1788972244.242344:0:102143:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972244.242347:0:102143:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [648:648:256:4294967295] 192.168.203.150@tcp LPNI seq info [648:648:8:4294967295] 00000400:00000200:3.0:1788972244.242351:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972244.242360:0:102143:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:3.0:1788972244.242365:0:102143:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:3.0:1788972244.242370:0:102143:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c4c67800. 02000000:00000010:2.0:1788972244.242375:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dd000. 00000800:00000200:3.0:1788972244.242377:0:102143:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000400:00000200:2.0:1788972244.242380:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972244.242383:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e0126370. 00000800:00000200:3.0:1788972244.242384:0:102143:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:3.0:1788972244.242388:0:102143:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c4c67800 type 1, nob 320 niov 1 nkiov 1 00000400:00000010:2.0:1788972244.242388:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972244.242390:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189596032, portal 25 00000400:00000010:2.0:1788972244.242394:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b110. 00000100:00000200:2.0:1788972244.242396:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189596032, offset 0 00000400:00000200:2.0:1788972244.242400:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:3.0:1788972244.242402:0:102143:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.242404:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:3.0:1788972244.242407:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09240. 00000400:00000200:2.0:1788972244.242407:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:2.0:1788972244.242410:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972244.242413:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:2.0:1788972244.242415:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000100:00000010:3.0:1788972244.242416:0:102143:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73d8a09300. 00000400:00000200:2.0:1788972244.242417:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:2.0:1788972244.242420:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [649:649:255:4294967295] 192.168.203.150@tcp LPNI seq info [649:649:7:4294967295] 00000400:00000200:2.0:1788972244.242423:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.242428:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972244.242432:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972244.242436:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63900. 00000800:00000200:2.0:1788972244.242440:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972244.242443:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972244.242446:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63900 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972244.242450:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:2.0:1788972244.242460:0:102142:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73e03dc000. 00000400:00000200:2.0:1788972244.242463:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:2.0:1788972244.242465:0:102142:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73e01260b0. 00000400:00000010:2.0:1788972244.242469:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972244.242471:0:102142:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189596288, portal 10 00000400:00000010:2.0:1788972244.242481:0:102142:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7c7b908. 00000100:00000200:2.0:1788972244.242484:0:102142:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189596288, offset 0 00000400:00000200:2.0:1788972244.242488:0:102142:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000800:00000200:3.0:1788972244.242492:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:2.0:1788972244.242492:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.242495:0:102142:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000800:00000010:3.0:1788972244.242496:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c4c67800. 00000400:00000200:2.0:1788972244.242498:0:102142:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1788972244.242500:0:102142:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:3.0:1788972244.242501:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972244.242502:0:102142:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1788972244.242504:0:102142:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:3.0:1788972244.242505:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:2.0:1788972244.242507:0:102142:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [650:650:254:4294967295] 192.168.203.150@tcp LPNI seq info [650:650:6:4294967295] 00000400:00000200:2.0:1788972244.242511:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.242516:0:102142:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:2.0:1788972244.242520:0:102142:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:2.0:1788972244.242522:0:102142:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73d0f63300. 00000800:00000200:2.0:1788972244.242526:0:102142:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:2.0:1788972244.242530:0:102142:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:2.0:1788972244.242532:0:102142:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73d0f63300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1788972244.242537:0:102142:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972244.247433:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c48733b8 00000400:00000010:3.0:1788972244.247441:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c48733b8. 00000100:00000200:3.0:1788972244.247461:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790e00 x1875872189596160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972244.248360:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972244.248365:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63900. 00000400:00000200:3.0:1788972244.248370:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972244.248376:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972244.248380:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b110 00000400:00000010:3.0:1788972244.248383:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b110. 00000100:00000200:3.0:1788972244.248390:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8790a80 x1875872189596032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972244.248462:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972244.248465:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73d0f63300. 00000400:00000200:3.0:1788972244.248604:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972244.248608:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972244.248612:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b908 00000400:00000010:3.0:1788972244.248615:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b908. 00000100:00000200:3.0:1788972244.248622:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73c8791c00 x1875872189596288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972244.254377:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254633:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972244.254640:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254644:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254648:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254652:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887814200 00000400:00000200:2.0:1788972244.254657:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254663:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x1fed [1] + 224 00000400:00000200:2.0:1788972244.254675:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972244.254679:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972244.254692:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790e00 x1875872189596160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972244.254805:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254809:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972244.254839:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254842:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254845:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254848:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887814180 00000400:00000200:2.0:1788972244.254851:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254856:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x1ffd [1] + 224 00000400:00000200:2.0:1788972244.254874:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972244.254877:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972244.254884:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8790a80 x1875872189596032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972244.254904:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254908:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972244.254913:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254915:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254918:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254921:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887814280 00000400:00000200:2.0:1788972244.254933:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972244.254937:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x200d [1] + 224 00000400:00000200:2.0:1788972244.254946:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972244.254950:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972244.254955:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73c8791c00 x1875872189596288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972244.254981:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e0126370. 00000400:00000200:2.0:1788972244.254986:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7b000 00000400:00000010:2.0:1788972244.254989:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7b000. 00000100:00000200:2.0:1788972244.254994:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790a80 x1875872189596032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972244.255009:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790a80 x1875872189596032/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1788972244.255033:0:102142:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73e01260b0. 00000400:00000200:2.0:1788972244.255036:0:102142:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7c7bd48 00000400:00000010:2.0:1788972244.255047:0:102142:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7c7bd48. 00000100:00000200:2.0:1788972244.255053:0:102142:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8791c00 x1875872189596288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1788972244.255062:0:102142:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8791c00 x1875872189596288/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1788972244.255081:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dd000. 02000000:00000010:2.0:1788972244.255088:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67e00. 00000100:00000010:2.0:1788972244.255092:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790a80. 02000000:00000010:2.0:1788972244.255096:0:102142:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73e03dc000. 02000000:00000010:2.0:1788972244.255099:0:102142:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67100. 00000100:00000010:2.0:1788972244.255102:0:102142:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8791c00. 00000400:00000010:3.0:1788972244.256079:0:102143:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c4436e18. 00000400:00000200:3.0:1788972244.256087:0:102143:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c4873990 00000400:00000010:3.0:1788972244.256089:0:102143:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c4873990. 00000100:00000200:3.0:1788972244.256098:0:102143:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73c8790e00 x1875872189596160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1788972244.256107:0:102143:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73c8790e00 x1875872189596160/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972260 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1788972244.256130:0:102143:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d107e000. 02000000:00000010:3.0:1788972244.256136:0:102143:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c4c67c00. 00000100:00000010:3.0:1788972244.256139:0:102143:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73c8790e00. 00000100:00000010:3.0:1788972245.265983:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09300. 00000100:00000010:2.0:1788972245.266044:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73d8a09240. 00000100:00000010:0.0:1788972249.362541:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88a80. 02000000:00000010:0.0:1788972249.362570:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cff200. 00000100:00000010:0.0:1788972249.362626:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a88000. 02000000:00000010:0.0:1788972249.362633:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffa00. 00000100:00000010:0.0:1788972249.362640:0:10833:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff9a73d1a8b480. 02000000:00000010:0.0:1788972249.362644:0:10833:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff9a73c7cffd00. 02000000:00000010:1.0:1788972249.362701:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4625000. 00000400:00000200:1.0:1788972249.362711:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972249.362715:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d738. 00000400:00000010:1.0:1788972249.362721:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972249.362727:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189596416, portal 25 00000400:00000010:1.0:1788972249.362732:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525330. 00000100:00000200:1.0:1788972249.362735:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1875872189596416, offset 0 00000400:00000200:1.0:1788972249.362744:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972249.362751:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972249.362758:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972249.362763:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972249.362766:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972249.362770:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972249.362772:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972249.362776:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [651:651:256:4294967295] 192.168.203.150@tcp LPNI seq info [651:651:8:4294967295] 00000400:00000200:1.0:1788972249.362780:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972249.362788:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972249.362792:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972249.362797:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b100. 00000800:00000200:1.0:1788972249.362802:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972249.362809:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972249.362836:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972249.362850:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000100:00000010:1.0:1788972249.362855:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467c60. 00000100:00000010:1.0:1788972249.362863:0:102140:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff9a73c7467c00. 02000000:00000010:1.0:1788972249.362872:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624c00. 00000400:00000200:1.0:1788972249.362876:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972249.362878:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1dec8. 00000400:00000010:1.0:1788972249.362881:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525b28. 00000100:00000200:1.0:1788972249.362899:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189596672, portal 10 00000400:00000010:1.0:1788972249.362904:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525110. 00000100:00000200:1.0:1788972249.362907:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189596672, offset 0 00000400:00000200:1.0:1788972249.362911:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972249.362914:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972249.362917:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972249.362920:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972249.362922:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972249.362923:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972249.362925:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972249.362928:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [652:652:255:4294967295] 192.168.203.150@tcp LPNI seq info [652:652:7:4294967295] 00000400:00000200:1.0:1788972249.362931:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972249.362936:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972249.362940:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972249.362942:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b300. 00000800:00000200:1.0:1788972249.362946:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972249.362949:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972249.362951:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972249.362956:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 02000000:00000010:1.0:1788972249.362972:0:102140:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff9a73d4624000. 00000400:00000200:1.0:1788972249.362975:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000010:1.0:1788972249.362977:0:102140:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff9a73c3f1d000. 00000400:00000010:1.0:1788972249.362979:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972249.362982:0:102140:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1875872189596544, portal 10 00000400:00000010:1.0:1788972249.362985:0:102140:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff9a73c7525220. 00000100:00000200:1.0:1788972249.362987:0:102140:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1875872189596544, offset 0 00000400:00000200:1.0:1788972249.362991:0:102140:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.203.150@tcp 00000400:00000200:1.0:1788972249.362993:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972249.362996:0:102140:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.203.150@tcp local destination 00000400:00000200:1.0:1788972249.362998:0:102140:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1788972249.363000:0:102140:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.203.50@tcp 00000400:00000200:1.0:1788972249.363006:0:102140:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.203.50@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1788972249.363008:0:102140:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.203.150@tcp 00000400:00000200:1.0:1788972249.363011:0:102140:0:(lib-move.c:1745:lnet_handle_send()) 192.168.203.50@tcp NI seq info: [653:653:254:4294967295] 192.168.203.150@tcp LPNI seq info [653:653:6:4294967295] 00000400:00000200:1.0:1788972249.363014:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:1.0:1788972249.363019:0:102140:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp:) -> 192.168.203.150@tcp(192.168.203.150@tcp:192.168.203.150@tcp) : PUT try# 0 00000800:00000200:1.0:1788972249.363023:0:102140:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.203.150@tcp 00000800:00000010:1.0:1788972249.363026:0:102140:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff9a73c2e1b000. 00000800:00000200:1.0:1788972249.363029:0:102140:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff9a73e0248600] -> 12345-192.168.203.150@tcp (5) 00000800:00000200:1.0:1788972249.363033:0:102140:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.203.150@tcp ip 192.168.203.150:988 00000800:00000200:1.0:1788972249.363035:0:102140:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff9a73c2e1b000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1788972249.363039:0:102140:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000800:00000200:3.0:1788972249.363142:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972249.363147:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b100. 00000400:00000200:3.0:1788972249.363155:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972249.363160:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972249.363165:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525330 00000400:00000010:3.0:1788972249.363167:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525330. 00000100:00000200:3.0:1788972249.363179:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88a80 x1875872189596416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972265 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972249.363204:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972249.363206:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b300. 00000400:00000200:3.0:1788972249.363208:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972249.363212:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972249.363214:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525110 00000400:00000010:3.0:1788972249.363216:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525110. 00000100:00000200:3.0:1788972249.363220:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a8b480 x1875872189596672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1788972249.363295:0:102135:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1788972249.363298:0:102135:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff9a73c2e1b000. 00000400:00000200:3.0:1788972249.363301:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972249.363308:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000400:00000200:3.0:1788972249.363314:0:102135:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525220 00000400:00000010:3.0:1788972249.363315:0:102135:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525220. 00000100:00000200:3.0:1788972249.363320:0:102135:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff9a73d1a88000 x1875872189596544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1788972249.367099:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.367110:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972249.367118:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.367265:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.367272:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.367278:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 25 MB=0x6aa1887814300 00000400:00000200:2.0:1788972249.367283:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.367291:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.203.150@tcp of length 224/224 into md 0x201d [1] + 224 00000400:00000200:2.0:1788972249.367307:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972249.367311:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972249.367325:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88a80 x1875872189596416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972249.367464:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d738. 00000400:00000200:1.0:1788972249.367471:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525c38 00000400:00000010:1.0:1788972249.367473:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525c38. 00000100:00000200:1.0:1788972249.367481:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88a80 x1875872189596416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972249.367491:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88a80 x1875872189596416/t0(0) o400->MGC192.168.203.150@tcp@192.168.203.150@tcp:26/25 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972249.367517:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4625000. 02000000:00000010:1.0:1788972249.367523:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cff200. 00000100:00000010:1.0:1788972249.367528:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88a80. 00000400:00000200:3.0:1788972249.368339:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972249.368345:0:102135:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:3.0:1788972249.368351:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972249.368355:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972249.368358:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972249.368362:0:102135:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887814400 00000400:00000200:3.0:1788972249.368366:0:102135:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:3.0:1788972249.368371:0:102135:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x2025 [1] + 224 00000400:00000200:3.0:1788972249.368380:0:102135:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1788972249.368389:0:102135:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:3.0:1788972249.368397:0:102135:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a8b480 x1875872189596672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972249.368554:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1dec8. 00000400:00000200:1.0:1788972249.368559:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525b28 00000400:00000010:1.0:1788972249.368560:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525b28. 00000100:00000200:1.0:1788972249.368566:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a8b480 x1875872189596672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972249.368576:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a8b480 x1875872189596672/t0(0) o400->lustre-MDT0001-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972249.368598:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624c00. 02000000:00000010:1.0:1788972249.368609:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffd00. 00000100:00000010:1.0:1788972249.368613:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a8b480. 00000400:00000200:2.0:1788972249.369733:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.369740:0:102134:0:(lib-move.c:4652:lnet_parse()) TRACE: 192.168.203.50@tcp(192.168.203.50@tcp) <- 192.168.203.150@tcp : PUT - for me 00000400:00000200:2.0:1788972249.369746:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.369749:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.369754:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.369758:0:102134:0:(lib-ptl.c:582:lnet_ptl_match_md()) Request from 12345-192.168.203.150@tcp of length 224 into portal 10 MB=0x6aa1887814380 00000400:00000200:2.0:1788972249.369763:0:102134:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.203.150@tcp to cpt 1 00000400:00000200:2.0:1788972249.369768:0:102134:0:(lib-ptl.c:208:lnet_try_match_md()) Incoming put index a from 12345-192.168.203.150@tcp of length 224/224 into md 0x202d [1] + 224 00000400:00000200:2.0:1788972249.369781:0:102134:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1788972249.369785:0:102134:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.203.50@tcp->192.168.203.150@tcp: PUT: OK 00000100:00000200:2.0:1788972249.369794:0:102134:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff9a73d1a88000 x1875872189596544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1788972249.369938:0:102140:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff9a73c3f1d000. 00000400:00000200:1.0:1788972249.369945:0:102140:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff9a73c7525a18 00000400:00000010:1.0:1788972249.369947:0:102140:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff9a73c7525a18. 00000100:00000200:1.0:1788972249.369954:0:102140:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff9a73d1a88000 x1875872189596544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1788972249.369963:0:102140:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff9a73d1a88000 x1875872189596544/t0(0) o400->lustre-MDT0000-mdc-ffff9a73d1aa9000@192.168.203.150@tcp:12/10 lens 224/224 e 0 to 0 dl 1788972265 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1788972249.369997:0:102140:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff9a73d4624000. 02000000:00000010:1.0:1788972249.370004:0:102140:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff9a73c7cffa00. 00000100:00000010:1.0:1788972249.370007:0:102140:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff9a73d1a88000. 00000001:02000400:2.0:1788972249.449329:0:116053:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: sanity-lnet test_304: @@@@@@ FAIL: Failed to unload modules before test execution 00000100:00000010:2.0:1788972250.386235:0:114511:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467c60. 00000100:00000010:3.0:1788972250.386640:0:111715:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff9a73c7467c00. Debug log: 4367 lines, 4367 kept, 0 dropped, 0 bad.